feat(observatory): add portable calculation profiles
This commit is contained in:
@@ -24,6 +24,7 @@ from k1link.observatory.run_preparations import (
|
||||
from k1link.observatory.setups import (
|
||||
LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
LABORATORY_SETUP_REGISTRY_SCHEMA,
|
||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
LaboratorySetupRegistry,
|
||||
LaboratorySetupRegistryError,
|
||||
)
|
||||
@@ -31,6 +32,7 @@ from k1link.observatory.setups import (
|
||||
__all__ = [
|
||||
"LABORATORY_SETUP_CATALOG_SCHEMA",
|
||||
"LABORATORY_SETUP_REGISTRY_SCHEMA",
|
||||
"OBSERVATORY_CALCULATION_PROFILE_SCHEMA",
|
||||
"MAX_RUN_PREPARATION_RECORDS",
|
||||
"MAX_RUN_PREPARATION_STORAGE_BYTES",
|
||||
"OBSERVATORY_RUN_PREPARATION_REQUEST_SCHEMA",
|
||||
|
||||
@@ -0,0 +1,523 @@
|
||||
"""Exact local composition for the portable M4.9 TRAVEL/TGS executor.
|
||||
|
||||
The server supplies only a sealed recorded-job identity. Source delivery and
|
||||
result upload remain implementations of the shared portable Worker ports; all
|
||||
filesystem locations, the compiled runner and its build seal are selected by
|
||||
reviewed Worker-local configuration.
|
||||
|
||||
This module does not register an executor or make a blocked runtime ready. It
|
||||
is the adapter that an installer may bind only after the release archive,
|
||||
compiled runner, executor image and local admission have all been sealed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_PROFILE_SHA256,
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
assemble_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.m49_portable_source import (
|
||||
M49_PORTABLE_STAGE_SCHEDULE,
|
||||
M49_PORTABLE_TGS_SEQUENCE,
|
||||
M49PortableSourceStage,
|
||||
materialize_m49_portable_source_from_worker_stage,
|
||||
validate_m49_portable_source_stage,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinition
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerExecutorAdapter,
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerResultPublisher,
|
||||
PortableWorkerRuntimeAdmission,
|
||||
PortableWorkerRuntimeCandidate,
|
||||
PortableWorkerRuntimeJobRejectedError,
|
||||
PortableWorkerRuntimePlan,
|
||||
PortableWorkerRuntimeUnavailableError,
|
||||
PortableWorkerSourceMaterializer,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
M49_COMPILED_RUNNER_BUILD_SCHEMA: Final = "missioncore.m49-tgs-portable-compiled-runner-build/v1"
|
||||
M49_PORTABLE_RUNNER_SOURCE_SHA256: Final = (
|
||||
"52813392aabd02efc5c2b8f7c22ed88e3ef4cc8ad3aafeba2792efe503e29fe9"
|
||||
)
|
||||
M49_PORTABLE_RUNNER_WRAPPER_SHA256: Final = (
|
||||
"2d6c32560682647f868e4ce4c2605749f17c60482a609f8c03ff951411f48ffb"
|
||||
)
|
||||
M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256: Final = (
|
||||
"7b412020f4d8392d1d1ed1b33beadc44140f0ea8f781e62dd69796042334300f"
|
||||
)
|
||||
M49_PORTABLE_RUNTIME_PHASES: Final = (
|
||||
"source-delivery",
|
||||
"camera-lidar-timeline-materializer",
|
||||
"portable-tgs-input-materializer",
|
||||
"portable-tgs-runner",
|
||||
"result-v2-assembler",
|
||||
"observatory-result-publisher",
|
||||
)
|
||||
M49_PORTABLE_COMPILED_RUNNER_ASSET_ID: Final = "m49-portable-compiled-runner"
|
||||
M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID: Final = "m49-portable-compiled-runner-build-seal"
|
||||
M49_PORTABLE_PROFILE_ASSET_ID: Final = "m49-portable-profile"
|
||||
M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID: Final = "travel-tgs-image"
|
||||
M49_PORTABLE_COMPILER_CONTRACT: Final = {
|
||||
"compiler": "g++",
|
||||
"language_standard": "c++17",
|
||||
"flags": ["-O3", "-DNDEBUG", "-pthread"],
|
||||
"travel_include": "/opt/travel/src/TRAVEL/cpp/travel/core",
|
||||
"eigen_include": "/usr/include/eigen3",
|
||||
}
|
||||
|
||||
_MAX_BUILD_SEAL_BYTES: Final = 128 * 1024
|
||||
_MAX_RUN_SECONDS: Final = 24 * 60 * 60
|
||||
|
||||
|
||||
class M49PortableExecutorError(PortableWorkerRuntimeUnavailableError):
|
||||
"""The local M4.9 executor installation or invocation is not exact."""
|
||||
|
||||
|
||||
class M49PortableRunnerInvoker(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
binary: Path,
|
||||
sequence: Path,
|
||||
schedule: Path,
|
||||
output: Path,
|
||||
timing: Path,
|
||||
workspace: Path,
|
||||
timeout_seconds: int,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49PortableRunnerInstallation:
|
||||
"""Worker-local paths bound to an exact build-only runner seal."""
|
||||
|
||||
profile_path: Path
|
||||
runner_binary_path: Path
|
||||
runner_build_seal_path: Path
|
||||
runner_build_seal_sha256: str
|
||||
output_parent: Path
|
||||
timeout_seconds: int = _MAX_RUN_SECONDS
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
profile = _exact_file(
|
||||
self.profile_path,
|
||||
M49_PORTABLE_PROFILE_SHA256,
|
||||
"portable M4.9 profile",
|
||||
)
|
||||
seal = _exact_file(
|
||||
self.runner_build_seal_path,
|
||||
self.runner_build_seal_sha256,
|
||||
"portable M4.9 runner build seal",
|
||||
)
|
||||
binary = _regular_file(self.runner_binary_path, "portable M4.9 compiled runner")
|
||||
document = _read_build_seal(seal)
|
||||
binary_row = _object(document["binary"], "portable M4.9 build-seal binary")
|
||||
if binary_row != {
|
||||
"file_name": "run_m49_tgs_portable",
|
||||
"format": "elf",
|
||||
"byte_length": binary.stat().st_size,
|
||||
"sha256": _sha256_file(binary),
|
||||
} or document["profile_sha256"] != _sha256_file(profile):
|
||||
raise M49PortableExecutorError(
|
||||
"portable M4.9 compiled runner differs from its build seal"
|
||||
)
|
||||
if not os.access(binary, os.X_OK):
|
||||
raise M49PortableExecutorError("portable M4.9 compiled runner is not executable")
|
||||
if not _is_elf(binary):
|
||||
raise M49PortableExecutorError("portable M4.9 compiled runner is not ELF")
|
||||
parent = self.output_parent.expanduser().absolute()
|
||||
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if parent.is_symlink() or not parent.is_dir():
|
||||
raise M49PortableExecutorError("portable M4.9 output root is unsafe")
|
||||
if (
|
||||
isinstance(self.timeout_seconds, bool)
|
||||
or not isinstance(self.timeout_seconds, int)
|
||||
or not 1 <= self.timeout_seconds <= _MAX_RUN_SECONDS
|
||||
):
|
||||
raise ValueError("portable M4.9 runner timeout is invalid")
|
||||
object.__setattr__(self, "profile_path", profile)
|
||||
object.__setattr__(self, "runner_binary_path", binary)
|
||||
object.__setattr__(self, "runner_build_seal_path", seal)
|
||||
object.__setattr__(self, "output_parent", parent)
|
||||
|
||||
@property
|
||||
def runner_binary_sha256(self) -> str:
|
||||
binary = _object(
|
||||
_read_build_seal(self.runner_build_seal_path)["binary"],
|
||||
"portable M4.9 build-seal binary",
|
||||
)
|
||||
return cast(str, binary["sha256"])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49PortableBoundSourceStage(PortableWorkerSourceStage):
|
||||
"""Claim-bound in-memory extension of the shared source-stage port."""
|
||||
|
||||
job: SealedObservatoryRecordedJob
|
||||
m49_stage: M49PortableSourceStage
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
PortableWorkerSourceStage.__post_init__(self)
|
||||
if (
|
||||
self.root != self.m49_stage.root
|
||||
or self.source_bundle_sha256 != self.job.source_bundle_sha256
|
||||
or self.source_capability_manifest_sha256 != self.job.source_capability_manifest_sha256
|
||||
or self.source_adapter_sha256 != self.job.source_adapter_sha256
|
||||
):
|
||||
raise M49PortableExecutorError("portable M4.9 bound source differs from its sealed job")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49PortableSourceMaterializerAdapter:
|
||||
"""Adapt exact Worker transport delivery to the M4.9 TGS source port."""
|
||||
|
||||
upstream: PortableWorkerSourceMaterializer
|
||||
profile_path: Path
|
||||
output_parent: Path
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
profile = _exact_file(
|
||||
self.profile_path,
|
||||
M49_PORTABLE_PROFILE_SHA256,
|
||||
"portable M4.9 profile",
|
||||
)
|
||||
parent = self.output_parent.expanduser().absolute()
|
||||
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if parent.is_symlink() or not parent.is_dir():
|
||||
raise M49PortableExecutorError("portable M4.9 source output root is unsafe")
|
||||
object.__setattr__(self, "profile_path", profile)
|
||||
object.__setattr__(self, "output_parent", parent)
|
||||
|
||||
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||
delivered = self.upstream.materialize(job)
|
||||
materialized = materialize_m49_portable_source_from_worker_stage(
|
||||
worker_stage=delivered,
|
||||
job=job,
|
||||
profile_path=self.profile_path,
|
||||
output_parent=self.output_parent,
|
||||
)
|
||||
return M49PortableBoundSourceStage(
|
||||
root=materialized.root,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||
source_adapter_sha256=job.source_adapter_sha256,
|
||||
job=job,
|
||||
m49_stage=materialized,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M49PortableProfileRunnerAdapter:
|
||||
"""Run the exact installed binary and assemble the deterministic result-v2."""
|
||||
|
||||
definition: PortableRunDefinition
|
||||
installation: M49PortableRunnerInstallation
|
||||
created_at_utc: Callable[[], str]
|
||||
invoker: M49PortableRunnerInvoker | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_verify_definition(self.definition)
|
||||
|
||||
def run(
|
||||
self,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
source: PortableWorkerSourceStage,
|
||||
) -> PortableWorkerResultDraft:
|
||||
if not isinstance(source, M49PortableBoundSourceStage):
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"portable M4.9 runner requires its claim-bound source stage"
|
||||
)
|
||||
job = source.job
|
||||
_verify_plan(plan, job=job, definition=self.definition)
|
||||
stage = validate_m49_portable_source_stage(source.root)
|
||||
_verify_installation_unchanged(self.installation)
|
||||
workspace = Path(
|
||||
tempfile.mkdtemp(prefix=".m49-portable-run-", dir=self.installation.output_parent)
|
||||
)
|
||||
try:
|
||||
output = workspace / "outputs"
|
||||
timing = workspace / "timing.tsv"
|
||||
invoker = self.invoker or _invoke_exact_runner
|
||||
invoker(
|
||||
binary=self.installation.runner_binary_path,
|
||||
sequence=stage.root / M49_PORTABLE_TGS_SEQUENCE,
|
||||
schedule=stage.root / M49_PORTABLE_STAGE_SCHEDULE,
|
||||
output=output,
|
||||
timing=timing,
|
||||
workspace=workspace,
|
||||
timeout_seconds=self.installation.timeout_seconds,
|
||||
)
|
||||
package = assemble_m49_portable_result(
|
||||
source_stage_root=stage.root,
|
||||
runner_output_root=output,
|
||||
runner_timing_path=timing,
|
||||
profile_path=self.installation.profile_path,
|
||||
output_parent=self.installation.output_parent / "result-packages",
|
||||
job=job,
|
||||
definition=self.definition,
|
||||
created_at_utc=self.created_at_utc(),
|
||||
)
|
||||
return PortableWorkerResultDraft(
|
||||
root=package.root,
|
||||
result_id=package.result_id,
|
||||
result_sha256=package.manifest.manifest_sha256,
|
||||
result_contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(workspace, ignore_errors=True)
|
||||
|
||||
|
||||
def compose_m49_portable_executor_adapter(
|
||||
*,
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
definition: PortableRunDefinition,
|
||||
admission: PortableWorkerRuntimeAdmission,
|
||||
source_transport: PortableWorkerSourceMaterializer,
|
||||
result_transport: PortableWorkerResultPublisher,
|
||||
installation: M49PortableRunnerInstallation,
|
||||
source_output_parent: Path,
|
||||
created_at_utc: Callable[[], str],
|
||||
invoker: M49PortableRunnerInvoker | None = None,
|
||||
) -> PortableWorkerExecutorAdapter:
|
||||
"""Compose shared Worker ports without changing their API or registry state."""
|
||||
|
||||
_verify_candidate_assets(candidate, installation)
|
||||
return PortableWorkerExecutorAdapter(
|
||||
candidate=candidate,
|
||||
definition=definition,
|
||||
admission=admission,
|
||||
source_materializer=M49PortableSourceMaterializerAdapter(
|
||||
upstream=source_transport,
|
||||
profile_path=installation.profile_path,
|
||||
output_parent=source_output_parent,
|
||||
),
|
||||
runner=M49PortableProfileRunnerAdapter(
|
||||
definition=definition,
|
||||
installation=installation,
|
||||
created_at_utc=created_at_utc,
|
||||
invoker=invoker,
|
||||
),
|
||||
publisher=result_transport,
|
||||
)
|
||||
|
||||
|
||||
def _invoke_exact_runner(
|
||||
*,
|
||||
binary: Path,
|
||||
sequence: Path,
|
||||
schedule: Path,
|
||||
output: Path,
|
||||
timing: Path,
|
||||
workspace: Path,
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
stdout = workspace / "runner.stdout.log"
|
||||
stderr = workspace / "runner.stderr.log"
|
||||
try:
|
||||
with stdout.open("xb") as stdout_stream, stderr.open("xb") as stderr_stream:
|
||||
completed = subprocess.run(
|
||||
[str(binary), str(sequence), str(schedule), str(output), str(timing)],
|
||||
cwd=workspace,
|
||||
env={"LANG": "C", "LC_ALL": "C", "TZ": "UTC"},
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=stdout_stream,
|
||||
stderr=stderr_stream,
|
||||
check=False,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
raise M49PortableExecutorError("portable M4.9 runner invocation failed") from exc
|
||||
if completed.returncode != 0:
|
||||
raise M49PortableExecutorError("portable M4.9 runner rejected its exact source stage")
|
||||
|
||||
|
||||
def _verify_definition(definition: PortableRunDefinition) -> None:
|
||||
components = {component.component_id: component for component in definition.components}
|
||||
profile = components.get("m49-tgs-portable-profile-v2")
|
||||
if (
|
||||
definition.setup_id != "m49-tgs-portable-v2"
|
||||
or definition.definition_id != "m49-tgs-portable"
|
||||
or definition.result_contract.contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||
or profile is None
|
||||
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
|
||||
):
|
||||
raise M49PortableExecutorError("portable M4.9 RunDefinition identity changed")
|
||||
|
||||
|
||||
def _verify_plan(
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
*,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
) -> None:
|
||||
if (
|
||||
plan.job_id != job.job_id
|
||||
or plan.setup_id != job.setup_id
|
||||
or plan.definition_sha256 != job.definition_sha256
|
||||
or plan.source_bundle_sha256 != job.source_bundle_sha256
|
||||
or plan.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
|
||||
or plan.result_contract_sha256 != M49_PORTABLE_RESULT_CONTRACT_SHA256
|
||||
or plan.phases != M49_PORTABLE_RUNTIME_PHASES
|
||||
or job.definition_sha256 != definition.definition_sha256
|
||||
):
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"portable M4.9 runtime plan differs from its sealed job"
|
||||
)
|
||||
|
||||
|
||||
def _verify_candidate_assets(
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
installation: M49PortableRunnerInstallation,
|
||||
) -> None:
|
||||
assets = {asset.asset_id: asset for asset in candidate.reusable_assets}
|
||||
binary = assets.get(M49_PORTABLE_COMPILED_RUNNER_ASSET_ID)
|
||||
build_seal = assets.get(M49_PORTABLE_COMPILED_RUNNER_BUILD_SEAL_ASSET_ID)
|
||||
profile = assets.get(M49_PORTABLE_PROFILE_ASSET_ID)
|
||||
image = assets.get(M49_PORTABLE_TRAVEL_IMAGE_ASSET_ID)
|
||||
if (
|
||||
binary is None
|
||||
or binary.kind != "local-file"
|
||||
or binary.sha256 != installation.runner_binary_sha256
|
||||
or binary.byte_length != installation.runner_binary_path.stat().st_size
|
||||
or build_seal is None
|
||||
or build_seal.kind != "local-file"
|
||||
or build_seal.sha256 != installation.runner_build_seal_sha256
|
||||
or build_seal.byte_length != installation.runner_build_seal_path.stat().st_size
|
||||
or profile is None
|
||||
or profile.sha256 != M49_PORTABLE_PROFILE_SHA256
|
||||
or image is None
|
||||
or image.kind != "container-image"
|
||||
or image.sha256 != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
|
||||
):
|
||||
raise M49PortableExecutorError(
|
||||
"portable M4.9 runtime candidate lacks exact installed runner assets"
|
||||
)
|
||||
|
||||
|
||||
def _verify_installation_unchanged(installation: M49PortableRunnerInstallation) -> None:
|
||||
_exact_file(
|
||||
installation.profile_path,
|
||||
M49_PORTABLE_PROFILE_SHA256,
|
||||
"portable M4.9 profile",
|
||||
)
|
||||
_exact_file(
|
||||
installation.runner_build_seal_path,
|
||||
installation.runner_build_seal_sha256,
|
||||
"portable M4.9 runner build seal",
|
||||
)
|
||||
binary = _regular_file(
|
||||
installation.runner_binary_path,
|
||||
"portable M4.9 compiled runner",
|
||||
)
|
||||
if (
|
||||
_sha256_file(binary) != installation.runner_binary_sha256
|
||||
or not os.access(binary, os.X_OK)
|
||||
or not _is_elf(binary)
|
||||
):
|
||||
raise M49PortableExecutorError("portable M4.9 installed runner changed")
|
||||
|
||||
|
||||
def _read_build_seal(path: Path) -> dict[str, object]:
|
||||
try:
|
||||
payload = path.read_bytes()
|
||||
decoded: object = json.loads(payload.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise M49PortableExecutorError("portable M4.9 build seal is unreadable") from exc
|
||||
if not 0 < len(payload) <= _MAX_BUILD_SEAL_BYTES:
|
||||
raise M49PortableExecutorError("portable M4.9 build seal size is invalid")
|
||||
document = _object(decoded, "portable M4.9 build seal")
|
||||
if (
|
||||
set(document)
|
||||
!= {
|
||||
"schema_version",
|
||||
"source_revision",
|
||||
"source_state",
|
||||
"build_image_sha256",
|
||||
"profile_sha256",
|
||||
"runner_source_sha256",
|
||||
"runner_wrapper_sha256",
|
||||
"compiler_contract",
|
||||
"binary",
|
||||
"authority",
|
||||
}
|
||||
or payload != canonical_json(document)
|
||||
or document["schema_version"] != M49_COMPILED_RUNNER_BUILD_SCHEMA
|
||||
or not isinstance(document["source_revision"], str)
|
||||
or len(document["source_revision"]) != 40
|
||||
or any(character not in "0123456789abcdef" for character in document["source_revision"])
|
||||
or document["source_state"] != "committed-snapshot"
|
||||
or document["build_image_sha256"] != M49_PORTABLE_TRAVEL_BUILD_IMAGE_SHA256
|
||||
or document["profile_sha256"] != M49_PORTABLE_PROFILE_SHA256
|
||||
or document["runner_source_sha256"] != M49_PORTABLE_RUNNER_SOURCE_SHA256
|
||||
or document["runner_wrapper_sha256"] != M49_PORTABLE_RUNNER_WRAPPER_SHA256
|
||||
or document["compiler_contract"] != M49_PORTABLE_COMPILER_CONTRACT
|
||||
or document["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise M49PortableExecutorError("portable M4.9 build seal identity changed")
|
||||
return document
|
||||
|
||||
|
||||
def _exact_file(path: Path, expected_sha256: str, label: str) -> Path:
|
||||
candidate = _regular_file(path, label)
|
||||
if _sha256_file(candidate) != expected_sha256:
|
||||
raise M49PortableExecutorError(f"{label} digest changed")
|
||||
return candidate
|
||||
|
||||
|
||||
def _regular_file(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M49PortableExecutorError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not os.path.samefile(candidate, resolved)
|
||||
):
|
||||
raise M49PortableExecutorError(f"{label} is unsafe")
|
||||
return resolved
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _is_elf(path: Path) -> bool:
|
||||
try:
|
||||
with path.open("rb") as stream:
|
||||
return stream.read(4) == b"\x7fELF"
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise M49PortableExecutorError(f"{label} must be an object")
|
||||
return cast(dict[str, object], value)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -133,9 +133,12 @@ class PortableRecordedQueueBindingService:
|
||||
) -> PortableRecordedSourceCapability:
|
||||
"""Return cheap authoritative compatibility without replay preparation."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
# Catalog projection must remain available for not-yet-installed
|
||||
# executors. Probe only resolves the immutable portable definition;
|
||||
# check/admit/submit still require a ready executor before source reads.
|
||||
portable = self._definitions.resolve(setup_id, definition_sha256)
|
||||
capability = self._source_service(portable).probe(source_session_id)
|
||||
if recorded.source_adapter_sha256 != capability.source_adapter_sha256:
|
||||
if portable.source_adapter.contract_sha256 != capability.source_adapter_sha256:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and source capability adapter identities disagree"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,601 @@
|
||||
"""Strict, path-free contracts for portable Observatory result packages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
|
||||
|
||||
PORTABLE_RESULT_PACKAGE_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-result-package/v1"
|
||||
)
|
||||
PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-result-package-identity/v1"
|
||||
)
|
||||
PORTABLE_RESULT_PUBLICATION_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-result-publication/v1"
|
||||
)
|
||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
|
||||
"missioncore.observatory-calculation-profile/v1"
|
||||
)
|
||||
RESULT_PACKAGE_MANIFEST_NAME: Final = "manifest.json"
|
||||
RESULT_DOCUMENT_ROLE: Final = "result-document"
|
||||
RESULT_PACKAGE_MANIFEST_ROLE: Final = "result-package-manifest"
|
||||
|
||||
_MAX_MANIFEST_BYTES: Final = 1024 * 1024
|
||||
_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
|
||||
_MAX_ARTIFACTS: Final = 128
|
||||
_MAX_ARTIFACT_BYTES: Final = (1 << 63) - 1
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_ROLE: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_PATH_COMPONENT: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_LAB_ID: Final = re.compile(r"^LAB [A-Z][A-Z0-9._-]{0,31}$")
|
||||
OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class PortableResultPublisherError(RuntimeError):
|
||||
"""Base class for verified portable result publication failures."""
|
||||
|
||||
|
||||
class PortableResultPackageIntegrityError(PortableResultPublisherError):
|
||||
"""The supplied package or one of its content identities is invalid."""
|
||||
|
||||
|
||||
class PortableResultPublicationBlockedError(PortableResultPublisherError):
|
||||
"""A required server-owned definition, policy, or validator is absent."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultArtifact:
|
||||
"""One confined, content-addressed package member."""
|
||||
|
||||
role: str
|
||||
relative_path: str
|
||||
media_type: str
|
||||
byte_length: int
|
||||
sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.role, _ROLE, "portable result artifact role")
|
||||
if self.role == RESULT_PACKAGE_MANIFEST_ROLE:
|
||||
raise ValueError("portable result artifact role is reserved")
|
||||
relative_artifact_path(self.relative_path)
|
||||
_media_type(self.media_type)
|
||||
if (
|
||||
not isinstance(self.byte_length, int)
|
||||
or isinstance(self.byte_length, bool)
|
||||
or not 0 <= self.byte_length <= _MAX_ARTIFACT_BYTES
|
||||
):
|
||||
raise ValueError("portable result artifact byte length is invalid")
|
||||
digest(self.sha256, "portable result artifact sha256")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"role": self.role,
|
||||
"relative_path": self.relative_path,
|
||||
"media_type": self.media_type,
|
||||
"byte_length": self.byte_length,
|
||||
"sha256": self.sha256,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultPackageManifest:
|
||||
"""Strict Worker-produced manifest with a separately hashed identity."""
|
||||
|
||||
identity_sha256: str
|
||||
created_at_utc: str
|
||||
job: dict[str, object]
|
||||
source: dict[str, object]
|
||||
run_definition: dict[str, object]
|
||||
result: dict[str, object]
|
||||
authority: dict[str, object]
|
||||
artifacts: tuple[PortableResultArtifact, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
digest(self.identity_sha256, "portable result package identity sha256")
|
||||
_timestamp(self.created_at_utc, "portable result package creation time")
|
||||
if self.authority != OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package is not observation-only"
|
||||
)
|
||||
if not 1 <= len(self.artifacts) <= _MAX_ARTIFACTS:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package artifact count is invalid"
|
||||
)
|
||||
roles = tuple(artifact.role for artifact in self.artifacts)
|
||||
paths = tuple(artifact.relative_path for artifact in self.artifacts)
|
||||
if roles != tuple(sorted(roles)) or len(set(roles)) != len(roles):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package artifacts are not canonically ordered"
|
||||
)
|
||||
if len(set(paths)) != len(paths):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package artifact paths are not unique"
|
||||
)
|
||||
result_documents = tuple(
|
||||
artifact for artifact in self.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
|
||||
)
|
||||
if (
|
||||
len(result_documents) != 1
|
||||
or result_documents[0].media_type != "application/json"
|
||||
or not 0 < result_documents[0].byte_length <= _MAX_RESULT_DOCUMENT_BYTES
|
||||
):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package requires one JSON result document"
|
||||
)
|
||||
if self.identity_sha256 != canonical_sha256(self.identity_document()):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package identity digest changed"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
result_id: str,
|
||||
created_at_utc: str,
|
||||
artifacts: Sequence[PortableResultArtifact],
|
||||
) -> PortableResultPackageManifest:
|
||||
"""Build the canonical package envelope used by a future Worker assembler."""
|
||||
|
||||
_pattern(result_id, _SESSION_ID, "portable result id")
|
||||
job_document = job_identity_document(job)
|
||||
source_document = source_identity_document(job)
|
||||
definition_document = run_definition_document(definition)
|
||||
result_document = result_identity_document(definition, result_id)
|
||||
authority: dict[str, object] = dict(OBSERVATION_ONLY_AUTHORITY)
|
||||
normalized_artifacts = tuple(
|
||||
sorted(artifacts, key=lambda artifact: artifact.role)
|
||||
)
|
||||
identity_document = _package_identity_document(
|
||||
created_at_utc=created_at_utc,
|
||||
job=job_document,
|
||||
source=source_document,
|
||||
run_definition=definition_document,
|
||||
result=result_document,
|
||||
authority=authority,
|
||||
artifacts=normalized_artifacts,
|
||||
)
|
||||
return cls(
|
||||
identity_sha256=canonical_sha256(identity_document),
|
||||
created_at_utc=created_at_utc,
|
||||
job=job_document,
|
||||
source=source_document,
|
||||
run_definition=definition_document,
|
||||
result=result_document,
|
||||
authority=authority,
|
||||
artifacts=normalized_artifacts,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, payload: bytes) -> PortableResultPackageManifest:
|
||||
if not 0 < len(payload) <= _MAX_MANIFEST_BYTES:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest size is invalid"
|
||||
)
|
||||
try:
|
||||
decoded: object = json.loads(payload.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest is not valid JSON"
|
||||
) from exc
|
||||
document = object_document(decoded, "portable result package manifest")
|
||||
exact_keys(
|
||||
document,
|
||||
{
|
||||
"schema_version",
|
||||
"identity_sha256",
|
||||
"created_at_utc",
|
||||
"job",
|
||||
"source",
|
||||
"run_definition",
|
||||
"result",
|
||||
"authority",
|
||||
"artifacts",
|
||||
},
|
||||
"portable result package manifest",
|
||||
)
|
||||
if document["schema_version"] != PORTABLE_RESULT_PACKAGE_SCHEMA:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest schema is invalid"
|
||||
)
|
||||
if payload != canonical_json(document):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest is not canonical JSON"
|
||||
)
|
||||
artifacts_value = document["artifacts"]
|
||||
if not isinstance(artifacts_value, list):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package artifacts are not an array"
|
||||
)
|
||||
artifacts = tuple(_artifact(value) for value in artifacts_value)
|
||||
try:
|
||||
return cls(
|
||||
identity_sha256=string(
|
||||
document["identity_sha256"],
|
||||
"portable result package identity sha256",
|
||||
),
|
||||
created_at_utc=string(
|
||||
document["created_at_utc"],
|
||||
"portable result package creation time",
|
||||
),
|
||||
job=object_document(document["job"], "portable result job identity"),
|
||||
source=object_document(
|
||||
document["source"], "portable result source identity"
|
||||
),
|
||||
run_definition=object_document(
|
||||
document["run_definition"],
|
||||
"portable result RunDefinition",
|
||||
),
|
||||
result=object_document(document["result"], "portable result identity"),
|
||||
authority=object_document(
|
||||
document["authority"], "portable result authority"
|
||||
),
|
||||
artifacts=artifacts,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest field is invalid"
|
||||
) from exc
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return _package_identity_document(
|
||||
created_at_utc=self.created_at_utc,
|
||||
job=self.job,
|
||||
source=self.source,
|
||||
run_definition=self.run_definition,
|
||||
result=self.result,
|
||||
authority=self.authority,
|
||||
artifacts=self.artifacts,
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
identity = self.identity_document()
|
||||
identity.pop("schema_version")
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||
"identity_sha256": self.identity_sha256,
|
||||
**identity,
|
||||
}
|
||||
|
||||
@property
|
||||
def canonical_bytes(self) -> bytes:
|
||||
return canonical_json(self.as_dict())
|
||||
|
||||
@property
|
||||
def manifest_sha256(self) -> str:
|
||||
return hashlib.sha256(self.canonical_bytes).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableCalculationProfilePolicy:
|
||||
"""Server-owned presentation identity bound to one exact RunDefinition."""
|
||||
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
lab_id: str
|
||||
display_name: str
|
||||
include_recorded_media: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.setup_id, _IDENTIFIER, "portable calculation profile setup id")
|
||||
_pattern(
|
||||
self.definition_id,
|
||||
_IDENTIFIER,
|
||||
"portable calculation profile definition id",
|
||||
)
|
||||
if (
|
||||
not isinstance(self.definition_version, int)
|
||||
or isinstance(self.definition_version, bool)
|
||||
or self.definition_version < 1
|
||||
):
|
||||
raise ValueError("portable calculation profile version is invalid")
|
||||
digest(
|
||||
self.definition_sha256,
|
||||
"portable calculation profile definition sha256",
|
||||
)
|
||||
if _LAB_ID.fullmatch(self.lab_id) is None:
|
||||
raise ValueError("portable calculation profile LAB id is invalid")
|
||||
_text(self.display_name, "portable calculation profile display name", maximum=160)
|
||||
if not isinstance(self.include_recorded_media, bool):
|
||||
raise ValueError("portable calculation profile media policy is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": self.setup_id,
|
||||
"display_name": self.display_name,
|
||||
"origin": "archived-definition",
|
||||
"definition_id": self.definition_id,
|
||||
"definition_version": self.definition_version,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
}
|
||||
|
||||
@property
|
||||
def identity_sha256(self) -> str:
|
||||
return canonical_sha256(self.as_dict())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableCalculationProfileRegistry:
|
||||
policies: tuple[PortableCalculationProfilePolicy, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
keys = tuple(
|
||||
(policy.setup_id, policy.definition_sha256) for policy in self.policies
|
||||
)
|
||||
if len(keys) != len(set(keys)):
|
||||
raise ValueError("portable calculation profile policies are not unique")
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
) -> PortableCalculationProfilePolicy:
|
||||
for policy in self.policies:
|
||||
if (
|
||||
policy.setup_id == definition.setup_id
|
||||
and policy.definition_sha256 == definition.definition_sha256
|
||||
):
|
||||
if (
|
||||
policy.definition_id != definition.definition_id
|
||||
or policy.definition_version != definition.version
|
||||
):
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"calculation profile policy disagrees with the RunDefinition"
|
||||
)
|
||||
return policy
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"exact calculation profile policy is not registered"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultValidationContext:
|
||||
"""Read-only inputs supplied to one exact result-contract validator."""
|
||||
|
||||
manifest: PortableResultPackageManifest
|
||||
job: ObservatoryRecordedJob
|
||||
definition: PortableRunDefinition
|
||||
result_document: Mapping[str, object]
|
||||
artifact_paths: Mapping[str, Path]
|
||||
|
||||
|
||||
type PortableResultContractValidator = Callable[[PortableResultValidationContext], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultContractValidatorRegistration:
|
||||
contract_sha256: str
|
||||
validator: PortableResultContractValidator
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
digest(self.contract_sha256, "portable result validator contract sha256")
|
||||
if not callable(self.validator):
|
||||
raise ValueError("portable result contract validator is not callable")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultContractValidatorRegistry:
|
||||
registrations: tuple[PortableResultContractValidatorRegistration, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
digests = tuple(registration.contract_sha256 for registration in self.registrations)
|
||||
if len(digests) != len(set(digests)):
|
||||
raise ValueError("portable result contract validators are not unique")
|
||||
|
||||
def resolve(self, contract_sha256: str) -> PortableResultContractValidator:
|
||||
digest(contract_sha256, "portable result contract sha256")
|
||||
for registration in self.registrations:
|
||||
if registration.contract_sha256 == contract_sha256:
|
||||
return registration.validator
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"exact portable result-contract validator is not installed"
|
||||
)
|
||||
|
||||
|
||||
def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||
return {
|
||||
"job_id": job.job_id,
|
||||
"request_sha256": job.request_sha256,
|
||||
"identity_sha256": job.identity_sha256,
|
||||
"submission_receipt_sha256": job.submission_receipt_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
}
|
||||
|
||||
|
||||
def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||
return {
|
||||
"session_id": job.source_session_id,
|
||||
"catalog_sha256": job.source_catalog_sha256,
|
||||
"bundle_sha256": job.source_bundle_sha256,
|
||||
"capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||
"adapter": {
|
||||
"adapter_id": job.source_adapter_id,
|
||||
"version": job.source_adapter_version,
|
||||
"adapter_sha256": job.source_adapter_sha256,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_definition_document(definition: PortableRunDefinition) -> dict[str, object]:
|
||||
return {
|
||||
**definition.identity_document(),
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
}
|
||||
|
||||
|
||||
def result_identity_document(
|
||||
definition: PortableRunDefinition,
|
||||
result_id: str,
|
||||
) -> dict[str, object]:
|
||||
contract = definition.result_contract
|
||||
return {
|
||||
"result_id": result_id,
|
||||
"result_schema": contract.result_schema,
|
||||
"result_kind": contract.result_kind,
|
||||
"result_contract_sha256": contract.contract_sha256,
|
||||
}
|
||||
|
||||
|
||||
def _package_identity_document(
|
||||
*,
|
||||
created_at_utc: str,
|
||||
job: Mapping[str, object],
|
||||
source: Mapping[str, object],
|
||||
run_definition: Mapping[str, object],
|
||||
result: Mapping[str, object],
|
||||
authority: Mapping[str, object],
|
||||
artifacts: Sequence[PortableResultArtifact],
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_IDENTITY_SCHEMA,
|
||||
"created_at_utc": created_at_utc,
|
||||
"job": dict(job),
|
||||
"source": dict(source),
|
||||
"run_definition": dict(run_definition),
|
||||
"result": dict(result),
|
||||
"authority": dict(authority),
|
||||
"artifacts": [artifact.as_dict() for artifact in artifacts],
|
||||
}
|
||||
|
||||
|
||||
def _artifact(value: object) -> PortableResultArtifact:
|
||||
row = object_document(value, "portable result artifact")
|
||||
exact_keys(
|
||||
row,
|
||||
{"role", "relative_path", "media_type", "byte_length", "sha256"},
|
||||
"portable result artifact",
|
||||
)
|
||||
byte_length = row["byte_length"]
|
||||
if not isinstance(byte_length, int) or isinstance(byte_length, bool):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact byte length is invalid"
|
||||
)
|
||||
return PortableResultArtifact(
|
||||
role=string(row["role"], "portable result artifact role"),
|
||||
relative_path=string(
|
||||
row["relative_path"],
|
||||
"portable result artifact relative path",
|
||||
),
|
||||
media_type=string(row["media_type"], "portable result artifact media type"),
|
||||
byte_length=byte_length,
|
||||
sha256=string(row["sha256"], "portable result artifact sha256"),
|
||||
)
|
||||
|
||||
|
||||
def relative_artifact_path(value: object) -> PurePosixPath:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= 512:
|
||||
raise ValueError("portable result artifact path is invalid")
|
||||
path = PurePosixPath(value)
|
||||
if (
|
||||
path.is_absolute()
|
||||
or path.as_posix() != value
|
||||
or len(path.parts) < 2
|
||||
or path.parts[0] != "artifacts"
|
||||
or any(
|
||||
part in {"", ".", ".."} or _PATH_COMPONENT.fullmatch(part) is None
|
||||
for part in path.parts
|
||||
)
|
||||
):
|
||||
raise ValueError("portable result artifact path is invalid")
|
||||
return path
|
||||
|
||||
|
||||
def _media_type(value: object) -> str:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not 3 <= len(value) <= 255
|
||||
or "/" not in value
|
||||
or value != value.strip()
|
||||
or any(ord(character) < 32 or ord(character) > 126 for character in value)
|
||||
):
|
||||
raise ValueError("portable result artifact media type is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def canonical_json(value: object) -> bytes:
|
||||
try:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package is not JSON-compatible"
|
||||
) from exc
|
||||
|
||||
|
||||
def object_document(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise PortableResultPackageIntegrityError(f"{label} must be an object")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||
if set(value) != expected:
|
||||
raise PortableResultPackageIntegrityError(f"{label} fields are invalid")
|
||||
|
||||
|
||||
def string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise PortableResultPackageIntegrityError(f"{label} must be text")
|
||||
return value
|
||||
|
||||
|
||||
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def digest(value: object, label: str) -> str:
|
||||
return _pattern(value, _SHA256, label)
|
||||
|
||||
|
||||
def _text(value: object, label: str, *, maximum: int) -> str:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not 1 <= len(value) <= maximum
|
||||
or value != value.strip()
|
||||
):
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _timestamp(value: object, label: str) -> str:
|
||||
from datetime import datetime
|
||||
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{label} is invalid")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{label} is invalid") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError(f"{label} must include a timezone")
|
||||
return value
|
||||
@@ -0,0 +1,809 @@
|
||||
"""Verified publication boundary for portable recorded Observatory results.
|
||||
|
||||
A Worker success acknowledgement is only a transport receipt. This module
|
||||
admits a result into the Session catalog only after a canonical result package
|
||||
is bound to the exact durable job, persisted source contracts, portable
|
||||
RunDefinition, result-contract validator, and observation-only authority.
|
||||
|
||||
The publisher deliberately has no production default validators or presentation
|
||||
policy. An unknown result contract or definition-bound calculation profile is
|
||||
therefore a hard publication blocker rather than an invitation to infer one from
|
||||
the current UI selection or a historical LAB label.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
from k1link.artifact_gateway import (
|
||||
ArtifactGatewayError,
|
||||
ArtifactManifest,
|
||||
ArtifactMember,
|
||||
CentralArtifactStore,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||
PORTABLE_RESULT_PUBLICATION_SCHEMA,
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
RESULT_PACKAGE_MANIFEST_NAME,
|
||||
RESULT_PACKAGE_MANIFEST_ROLE,
|
||||
PortableCalculationProfilePolicy,
|
||||
PortableCalculationProfileRegistry,
|
||||
PortableResultArtifact,
|
||||
PortableResultContractValidatorRegistry,
|
||||
PortableResultPackageIntegrityError,
|
||||
PortableResultPackageManifest,
|
||||
PortableResultPublicationBlockedError,
|
||||
PortableResultPublisherError,
|
||||
PortableResultValidationContext,
|
||||
canonical_json,
|
||||
exact_keys,
|
||||
job_identity_document,
|
||||
object_document,
|
||||
relative_artifact_path,
|
||||
result_identity_document,
|
||||
run_definition_document,
|
||||
source_identity_document,
|
||||
string,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
digest as validate_digest,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
|
||||
from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||
)
|
||||
from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
_COPY_CHUNK_BYTES = 1024 * 1024
|
||||
_FULL_ROUTE_LABEL = "полный маршрут и воспроизведение"
|
||||
_LEGACY_CANONICAL_RESULT = re.compile(
|
||||
r"^lab-v1-vegetation-shadow-[a-f0-9]{64}$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublishedPortableObservatoryResult:
|
||||
binding: LabSessionBinding
|
||||
package: PortableResultPackageManifest
|
||||
artifact_manifest: ArtifactManifest
|
||||
|
||||
|
||||
class PortableObservatoryResultPublisher:
|
||||
"""Validate, archive, and project one exact portable Worker result."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_store: SessionStore,
|
||||
artifact_store: CentralArtifactStore,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
calculation_profiles: PortableCalculationProfileRegistry,
|
||||
validators: PortableResultContractValidatorRegistry,
|
||||
) -> None:
|
||||
self._session_store = session_store
|
||||
self._artifact_store = artifact_store
|
||||
self._definitions = definitions
|
||||
self._calculation_profiles = calculation_profiles
|
||||
self._validators = validators
|
||||
|
||||
def publish(
|
||||
self,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
package_root: Path,
|
||||
) -> PublishedPortableObservatoryResult:
|
||||
"""Publish only a verified terminal package; exact retries are idempotent."""
|
||||
|
||||
_verify_terminal_job(job)
|
||||
try:
|
||||
definition = self._definitions.resolve(job.setup_id, job.definition_sha256)
|
||||
except (PortableRunDefinitionRegistryError, ValueError) as exc:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"recorded job RunDefinition is not in the portable registry"
|
||||
) from exc
|
||||
_verify_definition_job_identity(definition, job)
|
||||
profile = self._calculation_profiles.resolve(definition)
|
||||
validator = self._validators.resolve(definition.result_contract.contract_sha256)
|
||||
|
||||
root, manifest_path, package = _read_package(package_root)
|
||||
_verify_package_identity(package, job=job, definition=definition)
|
||||
if package.manifest_sha256 != job.result_sha256 or root.name != job.result_sha256:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package content address disagrees with queue success"
|
||||
)
|
||||
source_summary = self._verify_source(job, definition)
|
||||
artifact_paths = _verify_package_artifacts(root, package.artifacts)
|
||||
result_document = _read_canonical_result_document(
|
||||
artifact_paths[RESULT_DOCUMENT_ROLE]
|
||||
)
|
||||
context = PortableResultValidationContext(
|
||||
manifest=package,
|
||||
job=job,
|
||||
definition=definition,
|
||||
result_document=result_document,
|
||||
artifact_paths=artifact_paths,
|
||||
)
|
||||
try:
|
||||
validator(context)
|
||||
except PortableResultPublisherError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result document failed its exact contract validator"
|
||||
) from exc
|
||||
|
||||
artifact_manifest = self._archive_package(
|
||||
job=job,
|
||||
package=package,
|
||||
manifest_path=manifest_path,
|
||||
artifact_paths=artifact_paths,
|
||||
profile=profile,
|
||||
)
|
||||
provenance = _publication_provenance(
|
||||
job=job,
|
||||
definition=definition,
|
||||
package=package,
|
||||
artifact_manifest=artifact_manifest,
|
||||
profile=profile,
|
||||
)
|
||||
try:
|
||||
binding = self._session_store.publish_lab_instance(
|
||||
session_id=cast(str, job.result_id),
|
||||
source_session_id=job.source_session_id,
|
||||
display_name=_portable_result_display_name(source_summary),
|
||||
lab_id=profile.lab_id,
|
||||
result_kind=definition.result_contract.result_kind,
|
||||
result_id=cast(str, job.result_id),
|
||||
config_sha256=definition.definition_sha256,
|
||||
run_created_at_utc=job.updated_at_utc,
|
||||
replay_capability=None,
|
||||
provenance=provenance,
|
||||
include_recorded_media=profile.include_recorded_media,
|
||||
expected_source_catalog_sha256=job.source_catalog_sha256,
|
||||
)
|
||||
except (SessionIntegrityError, ValueError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result could not be projected as immutable LAB provenance"
|
||||
) from exc
|
||||
return PublishedPortableObservatoryResult(
|
||||
binding=binding,
|
||||
package=package,
|
||||
artifact_manifest=artifact_manifest,
|
||||
)
|
||||
|
||||
def _verify_source(
|
||||
self,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
) -> SessionSummary:
|
||||
try:
|
||||
detail, catalog_sha256 = (
|
||||
self._session_store.get_session_with_catalog_snapshot(
|
||||
job.source_session_id
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result source session is unavailable"
|
||||
) from exc
|
||||
if (
|
||||
detail.summary.session_id != job.source_session_id
|
||||
or detail.summary.lab is not None
|
||||
or catalog_sha256 != job.source_catalog_sha256
|
||||
):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result source catalog changed after admission"
|
||||
)
|
||||
_verify_source_documents(
|
||||
self._session_store.data_dir,
|
||||
job=job,
|
||||
definition=definition,
|
||||
)
|
||||
return detail.summary
|
||||
|
||||
def _archive_package(
|
||||
self,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
package: PortableResultPackageManifest,
|
||||
manifest_path: Path,
|
||||
artifact_paths: Mapping[str, Path],
|
||||
profile: PortableCalculationProfilePolicy,
|
||||
) -> ArtifactManifest:
|
||||
members = [
|
||||
_publish_exact_member(
|
||||
self._artifact_store,
|
||||
role=RESULT_PACKAGE_MANIFEST_ROLE,
|
||||
media_type="application/json",
|
||||
source=manifest_path,
|
||||
expected_sha256=cast(str, job.result_sha256),
|
||||
expected_byte_length=len(package.canonical_bytes),
|
||||
)
|
||||
]
|
||||
for artifact in package.artifacts:
|
||||
members.append(
|
||||
_publish_exact_member(
|
||||
self._artifact_store,
|
||||
role=artifact.role,
|
||||
media_type=artifact.media_type,
|
||||
source=artifact_paths[artifact.role],
|
||||
expected_sha256=artifact.sha256,
|
||||
expected_byte_length=artifact.byte_length,
|
||||
)
|
||||
)
|
||||
try:
|
||||
archived = self._artifact_store.publish_manifest(
|
||||
artifact_type="observatory-portable-result",
|
||||
subject_id=cast(str, job.result_id),
|
||||
members=members,
|
||||
metadata={
|
||||
"package-sha256": cast(str, job.result_sha256),
|
||||
"package-identity-sha256": package.identity_sha256,
|
||||
"job-id": job.job_id,
|
||||
"job-identity-sha256": job.identity_sha256,
|
||||
"definition-sha256": job.definition_sha256,
|
||||
"source-bundle-sha256": job.source_bundle_sha256,
|
||||
"result-contract-sha256": string(
|
||||
package.result["result_contract_sha256"],
|
||||
"portable result contract sha256",
|
||||
),
|
||||
"calculation-profile-sha256": profile.identity_sha256,
|
||||
},
|
||||
created_at_utc=job.updated_at_utc,
|
||||
)
|
||||
verified = self._artifact_store.read_manifest(archived.manifest_id)
|
||||
except (ArtifactGatewayError, OSError, ValueError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package could not be archived immutably"
|
||||
) from exc
|
||||
if verified != archived:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact manifest changed after publication"
|
||||
)
|
||||
return archived
|
||||
|
||||
|
||||
def _portable_result_display_name(source: SessionSummary) -> str:
|
||||
"""Keep the result label source-owned; profile provenance is rendered separately."""
|
||||
|
||||
candidate = f"{source.display_name} · {_FULL_ROUTE_LABEL}"
|
||||
return candidate if len(candidate) <= 160 else source.display_name
|
||||
|
||||
|
||||
def resolve_published_portable_calculation_profile(
|
||||
summary: SessionSummary,
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
calculation_profiles: PortableCalculationProfileRegistry,
|
||||
) -> dict[str, object] | None:
|
||||
"""Resolve only an exact, immutable portable publication profile.
|
||||
|
||||
Catalog projection must never infer profile identity from a selected setup,
|
||||
a display label, or a current definition with the same setup id. Any drift
|
||||
in the stored publication provenance therefore makes the profile absent.
|
||||
"""
|
||||
|
||||
binding = summary.lab
|
||||
if binding is None:
|
||||
return None
|
||||
try:
|
||||
provenance = object_document(
|
||||
binding.provenance,
|
||||
"portable result publication provenance",
|
||||
)
|
||||
exact_keys(
|
||||
provenance,
|
||||
{
|
||||
"schema_version",
|
||||
"authority",
|
||||
"calculation_profile",
|
||||
"calculation_profile_sha256",
|
||||
"job",
|
||||
"source",
|
||||
"run_definition",
|
||||
"result_package",
|
||||
"storage",
|
||||
"method",
|
||||
},
|
||||
"portable result publication provenance",
|
||||
)
|
||||
profile_document = object_document(
|
||||
provenance["calculation_profile"],
|
||||
"portable calculation profile",
|
||||
)
|
||||
run_definition = object_document(
|
||||
provenance["run_definition"],
|
||||
"portable result RunDefinition",
|
||||
)
|
||||
source = object_document(provenance["source"], "portable result source")
|
||||
setup_id = string(profile_document.get("setup_id"), "portable setup id")
|
||||
definition_sha256 = string(
|
||||
run_definition.get("definition_sha256"),
|
||||
"portable RunDefinition sha256",
|
||||
)
|
||||
definition = definitions.resolve(setup_id, definition_sha256)
|
||||
profile = calculation_profiles.resolve(definition)
|
||||
if (
|
||||
provenance["schema_version"] != PORTABLE_RESULT_PUBLICATION_SCHEMA
|
||||
or provenance["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
or provenance["calculation_profile_sha256"] != profile.identity_sha256
|
||||
or profile_document != profile.as_dict()
|
||||
or canonical_sha256(profile_document) != profile.identity_sha256
|
||||
or run_definition != run_definition_document(definition)
|
||||
or binding.session_id != summary.session_id
|
||||
or binding.result_id != summary.session_id
|
||||
or binding.source_session_id != source.get("session_id")
|
||||
or binding.lab_id != profile.lab_id
|
||||
or binding.config_sha256 != definition.definition_sha256
|
||||
or binding.result_kind != definition.result_contract.result_kind
|
||||
):
|
||||
return None
|
||||
return profile.as_dict()
|
||||
except (
|
||||
KeyError,
|
||||
PortableResultPublisherError,
|
||||
PortableRunDefinitionRegistryError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
):
|
||||
return None
|
||||
|
||||
|
||||
def _verify_terminal_job(job: ObservatoryRecordedJob) -> None:
|
||||
if (
|
||||
job.state != "succeeded"
|
||||
or job.result_id is None
|
||||
or job.result_sha256 is None
|
||||
or job.terminal_code != "result-sealed"
|
||||
or job.terminal_claim_token_sha256 is None
|
||||
or job.claim_generation < 1
|
||||
or job.active_claim_token is not None
|
||||
or job.active_claimant_id is not None
|
||||
):
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"recorded job has no exact terminal Worker result receipt"
|
||||
)
|
||||
if _LEGACY_CANONICAL_RESULT.fullmatch(job.result_id) is not None:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"legacy canonical result namespace is immutable and reserved"
|
||||
)
|
||||
|
||||
|
||||
def _verify_definition_job_identity(
|
||||
definition: PortableRunDefinition,
|
||||
job: ObservatoryRecordedJob,
|
||||
) -> None:
|
||||
if not definition.executor.ready:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"portable result RunDefinition executor is not installed"
|
||||
)
|
||||
try:
|
||||
recorded = definition.to_recorded_run_definition()
|
||||
except Exception as exc:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"portable result RunDefinition cannot produce a queue identity"
|
||||
) from exc
|
||||
expected = (
|
||||
recorded.setup_id,
|
||||
recorded.definition_id,
|
||||
recorded.definition_version,
|
||||
recorded.definition_sha256,
|
||||
recorded.source_adapter_id,
|
||||
recorded.source_adapter_version,
|
||||
recorded.source_adapter_sha256,
|
||||
recorded.executor_release_id,
|
||||
recorded.executor_release_sha256,
|
||||
recorded.executor_image_sha256,
|
||||
recorded.model_release_ids,
|
||||
recorded.model_manifest_sha256,
|
||||
recorded.resource_profile_id,
|
||||
recorded.resource_profile_sha256,
|
||||
recorded.checkpoint_policy,
|
||||
recorded.allowed_checkpoints,
|
||||
)
|
||||
actual = (
|
||||
job.setup_id,
|
||||
job.definition_id,
|
||||
job.definition_version,
|
||||
job.definition_sha256,
|
||||
job.source_adapter_id,
|
||||
job.source_adapter_version,
|
||||
job.source_adapter_sha256,
|
||||
job.executor_release_id,
|
||||
job.executor_release_sha256,
|
||||
job.executor_image_sha256,
|
||||
job.model_release_ids,
|
||||
job.model_manifest_sha256,
|
||||
job.resource_profile_id,
|
||||
job.resource_profile_sha256,
|
||||
job.checkpoint_policy,
|
||||
job.allowed_checkpoints,
|
||||
)
|
||||
if actual != expected or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"recorded job and portable RunDefinition identities disagree"
|
||||
)
|
||||
|
||||
|
||||
def _read_package(
|
||||
package_root: Path,
|
||||
) -> tuple[Path, Path, PortableResultPackageManifest]:
|
||||
candidate = package_root.expanduser().absolute()
|
||||
try:
|
||||
root_metadata = candidate.lstat()
|
||||
root = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package root is unavailable"
|
||||
) from exc
|
||||
if stat.S_ISLNK(root_metadata.st_mode) or not stat.S_ISDIR(root_metadata.st_mode):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package root must be a regular directory"
|
||||
)
|
||||
manifest_path = root / RESULT_PACKAGE_MANIFEST_NAME
|
||||
try:
|
||||
metadata = manifest_path.lstat()
|
||||
payload = manifest_path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest is unavailable"
|
||||
) from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest must be a regular file"
|
||||
)
|
||||
package = PortableResultPackageManifest.from_bytes(payload)
|
||||
if hashlib.sha256(payload).hexdigest() != package.manifest_sha256:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package manifest digest changed"
|
||||
)
|
||||
return root, manifest_path, package
|
||||
|
||||
|
||||
def _verify_package_identity(
|
||||
package: PortableResultPackageManifest,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
) -> None:
|
||||
if package.job != job_identity_document(job):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package is bound to another queue job"
|
||||
)
|
||||
if package.source != source_identity_document(job):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package is bound to another source"
|
||||
)
|
||||
if package.run_definition != run_definition_document(definition):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package is bound to another RunDefinition"
|
||||
)
|
||||
if package.result != result_identity_document(definition, cast(str, job.result_id)):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package violates its sealed result contract"
|
||||
)
|
||||
if package.authority != definition.authority.as_dict():
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result package authority changed"
|
||||
)
|
||||
|
||||
|
||||
def _verify_source_documents(
|
||||
data_dir: Path,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
) -> None:
|
||||
root = data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
try:
|
||||
metadata = root.lstat()
|
||||
except OSError as exc:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"admitted portable source documents are unavailable"
|
||||
) from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable source document root is unsafe"
|
||||
)
|
||||
bundle = _read_content_addressed_document(root, job.source_bundle_sha256)
|
||||
capability = _read_content_addressed_document(
|
||||
root,
|
||||
job.source_capability_manifest_sha256,
|
||||
)
|
||||
exact_keys(
|
||||
bundle,
|
||||
{
|
||||
"schema_version",
|
||||
"source_session_id",
|
||||
"source_catalog_sha256",
|
||||
"plugin_id",
|
||||
"archive_id",
|
||||
"source_adapter",
|
||||
"sources",
|
||||
"spatial_replay",
|
||||
"camera",
|
||||
"authority",
|
||||
},
|
||||
"portable source bundle",
|
||||
)
|
||||
exact_keys(
|
||||
capability,
|
||||
{
|
||||
"schema_version",
|
||||
"source_session_id",
|
||||
"source_catalog_sha256",
|
||||
"source_bundle_sha256",
|
||||
"source_adapter_sha256",
|
||||
"modalities",
|
||||
"camera_profile",
|
||||
"calibration",
|
||||
"authority",
|
||||
},
|
||||
"portable source capability",
|
||||
)
|
||||
expected_adapter = {
|
||||
"id": job.source_adapter_id,
|
||||
"version": job.source_adapter_version,
|
||||
"sha256": job.source_adapter_sha256,
|
||||
}
|
||||
if (
|
||||
bundle["schema_version"] != PORTABLE_SOURCE_BUNDLE_SCHEMA
|
||||
or bundle["source_session_id"] != job.source_session_id
|
||||
or bundle["source_catalog_sha256"] != job.source_catalog_sha256
|
||||
or bundle["plugin_id"] != definition.source_requirements.plugin_id
|
||||
or bundle["archive_id"] != definition.source_requirements.archive_id
|
||||
or bundle["source_adapter"] != expected_adapter
|
||||
or bundle["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
or capability["schema_version"] != PORTABLE_SOURCE_CAPABILITY_SCHEMA
|
||||
or capability["source_session_id"] != job.source_session_id
|
||||
or capability["source_catalog_sha256"] != job.source_catalog_sha256
|
||||
or capability["source_bundle_sha256"] != job.source_bundle_sha256
|
||||
or capability["source_adapter_sha256"] != job.source_adapter_sha256
|
||||
or capability["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"persisted portable source contract disagrees with the queue job"
|
||||
)
|
||||
|
||||
|
||||
def _read_content_addressed_document(root: Path, document_sha256: str) -> dict[str, object]:
|
||||
validate_digest(document_sha256, "portable source document sha256")
|
||||
path = root / f"{document_sha256}.json"
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
payload = path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise PortableResultPublicationBlockedError(
|
||||
"an admitted portable source document is unavailable"
|
||||
) from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or hashlib.sha256(payload).hexdigest() != document_sha256
|
||||
):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"admitted portable source document digest changed"
|
||||
)
|
||||
try:
|
||||
decoded: object = json.loads(payload.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"admitted portable source document is not valid JSON"
|
||||
) from exc
|
||||
document = object_document(decoded, "portable source document")
|
||||
if payload != canonical_json(document):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"admitted portable source document is not canonical JSON"
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
def _verify_package_artifacts(
|
||||
root: Path,
|
||||
artifacts: Sequence[PortableResultArtifact],
|
||||
) -> dict[str, Path]:
|
||||
resolved: dict[str, Path] = {}
|
||||
for artifact in artifacts:
|
||||
path = _resolve_package_member(root, artifact.relative_path)
|
||||
digest, byte_length = _hash_file(path)
|
||||
if digest != artifact.sha256 or byte_length != artifact.byte_length:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
f"portable result artifact content changed: {artifact.role}"
|
||||
)
|
||||
resolved[artifact.role] = path
|
||||
return resolved
|
||||
|
||||
|
||||
def _resolve_package_member(root: Path, relative_path: str) -> Path:
|
||||
portable = relative_artifact_path(relative_path)
|
||||
candidate = root.joinpath(*portable.parts)
|
||||
current = root
|
||||
try:
|
||||
for part in portable.parts:
|
||||
current = current / part
|
||||
metadata = current.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact path contains a symlink"
|
||||
)
|
||||
resolved = candidate.resolve(strict=True)
|
||||
metadata = candidate.lstat()
|
||||
except PortableResultPublisherError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact is unavailable"
|
||||
) from exc
|
||||
if not resolved.is_relative_to(root) or not stat.S_ISREG(metadata.st_mode):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact escapes its package"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _read_canonical_result_document(path: Path) -> dict[str, object]:
|
||||
try:
|
||||
payload = path.read_bytes()
|
||||
decoded: object = json.loads(payload.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result document is not valid JSON"
|
||||
) from exc
|
||||
document = object_document(decoded, "portable result document")
|
||||
if payload != canonical_json(document):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result document is not canonical JSON"
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
def _publish_exact_member(
|
||||
store: CentralArtifactStore,
|
||||
*,
|
||||
role: str,
|
||||
media_type: str,
|
||||
source: Path,
|
||||
expected_sha256: str,
|
||||
expected_byte_length: int,
|
||||
) -> ArtifactMember:
|
||||
try:
|
||||
published = store.publish_file(source)
|
||||
except (ArtifactGatewayError, OSError, ValueError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
f"portable result artifact could not be archived: {role}"
|
||||
) from exc
|
||||
if (
|
||||
published.sha256 != expected_sha256
|
||||
or published.byte_length != expected_byte_length
|
||||
):
|
||||
raise PortableResultPackageIntegrityError(
|
||||
f"portable result artifact changed while it was archived: {role}"
|
||||
)
|
||||
return ArtifactMember(
|
||||
role=role,
|
||||
media_type=media_type,
|
||||
sha256=published.sha256,
|
||||
byte_length=published.byte_length,
|
||||
)
|
||||
|
||||
|
||||
def _publication_provenance(
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
package: PortableResultPackageManifest,
|
||||
artifact_manifest: ArtifactManifest,
|
||||
profile: PortableCalculationProfilePolicy,
|
||||
) -> dict[str, object]:
|
||||
result_document = next(
|
||||
artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
|
||||
)
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_PUBLICATION_SCHEMA,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
"calculation_profile": profile.as_dict(),
|
||||
"calculation_profile_sha256": profile.identity_sha256,
|
||||
"job": job_identity_document(job),
|
||||
"source": source_identity_document(job),
|
||||
"run_definition": run_definition_document(definition),
|
||||
"result_package": {
|
||||
"schema_version": PORTABLE_RESULT_PACKAGE_SCHEMA,
|
||||
"manifest_sha256": package.manifest_sha256,
|
||||
"identity_sha256": package.identity_sha256,
|
||||
"artifact_manifest_id": artifact_manifest.manifest_id,
|
||||
"result_document_sha256": result_document.sha256,
|
||||
"artifacts": [artifact.as_dict() for artifact in package.artifacts],
|
||||
},
|
||||
"storage": {
|
||||
"mode": "central-content-addressed-artifact-store",
|
||||
"include_recorded_media": profile.include_recorded_media,
|
||||
"replay_capability": None,
|
||||
},
|
||||
"method": _laboratory_method(job, definition),
|
||||
}
|
||||
|
||||
|
||||
def _laboratory_method(
|
||||
job: ObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
) -> dict[str, object]:
|
||||
components: list[dict[str, object]] = [
|
||||
{
|
||||
"kind": "source",
|
||||
"name": job.source_session_id,
|
||||
"version": f"{job.source_adapter_id}/v{job.source_adapter_version}",
|
||||
"role": "immutable admitted K1 source bundle",
|
||||
"identity_sha256": job.source_bundle_sha256,
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": definition.definition_id,
|
||||
"version": f"v{definition.version}",
|
||||
"role": "portable laboratory RunDefinition",
|
||||
"identity_sha256": definition.definition_sha256,
|
||||
},
|
||||
{
|
||||
"kind": "runtime",
|
||||
"name": job.executor_release_id,
|
||||
"version": "sealed-release",
|
||||
"role": "portable Worker executor",
|
||||
"identity_sha256": job.executor_release_sha256,
|
||||
},
|
||||
{
|
||||
"kind": "runtime",
|
||||
"name": job.resource_profile_id,
|
||||
"version": "sealed-resource-profile",
|
||||
"role": "exclusive Worker resource contract",
|
||||
"identity_sha256": job.resource_profile_sha256,
|
||||
},
|
||||
]
|
||||
components.extend(
|
||||
{
|
||||
"kind": "model",
|
||||
"name": model.release_id,
|
||||
"version": model.revision or "sealed-artifacts",
|
||||
"role": "portable inference model",
|
||||
"identity_sha256": model.identity_sha256,
|
||||
}
|
||||
for model in definition.models
|
||||
)
|
||||
return {
|
||||
"schema_version": "missioncore.laboratory-method/v1",
|
||||
"completeness": "complete",
|
||||
"execution_class": "hybrid" if definition.models else "deterministic",
|
||||
"pipeline_id": definition.definition_id,
|
||||
"components": components,
|
||||
}
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> tuple[str, int]:
|
||||
digest = hashlib.sha256()
|
||||
byte_length = 0
|
||||
try:
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(_COPY_CHUNK_BYTES):
|
||||
digest.update(chunk)
|
||||
byte_length += len(chunk)
|
||||
except OSError as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"portable result artifact could not be read"
|
||||
) from exc
|
||||
return digest.hexdigest(), byte_length
|
||||
@@ -510,26 +510,46 @@ class PortableRunDefinition:
|
||||
by_kind: dict[str, list[ImmutableComponentIdentity]] = {}
|
||||
for component in self.components:
|
||||
by_kind.setdefault(component.kind, []).append(component)
|
||||
for required_kind in (
|
||||
"calibration",
|
||||
# Calibration is common to every admitted K1 source. Learned-model
|
||||
# definitions additionally keep the original profile/runner/FOV seals;
|
||||
# algorithm-only definitions such as M4.9 may remain projectable while
|
||||
# their portable runner is not installed yet.
|
||||
required_kinds: tuple[str, ...] = ("calibration",)
|
||||
if self.models:
|
||||
required_kinds += (
|
||||
"profile",
|
||||
"runner",
|
||||
"valid-fov-identity",
|
||||
"valid-fov-mask",
|
||||
)
|
||||
for required_kind in required_kinds:
|
||||
if len(by_kind.get(required_kind, [])) != 1:
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
f"portable definition requires exactly one {required_kind} component"
|
||||
)
|
||||
for singleton_kind in (
|
||||
"profile",
|
||||
"runner",
|
||||
"valid-fov-identity",
|
||||
"valid-fov-mask",
|
||||
):
|
||||
if len(by_kind.get(required_kind, [])) != 1:
|
||||
if len(by_kind.get(singleton_kind, [])) > 1:
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
f"portable definition requires exactly one {required_kind} component"
|
||||
f"portable definition allows at most one {singleton_kind} component"
|
||||
)
|
||||
if not self.models and self.executor.ready and len(by_kind.get("runner", [])) != 1:
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
"ready portable definition requires exactly one runner component"
|
||||
)
|
||||
calibration = by_kind["calibration"][0]
|
||||
if calibration.sha256 != self.source_requirements.calibration_identity_sha256:
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
"source capability and runtime calibration identities disagree"
|
||||
)
|
||||
model_ids = [model.release_id for model in self.models]
|
||||
if not model_ids or model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
|
||||
if model_ids != sorted(model_ids) or len(model_ids) != len(set(model_ids)):
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
"models must be non-empty, unique, and canonically ordered"
|
||||
"models must be unique and canonically ordered"
|
||||
)
|
||||
if self.executor.contour_id != self.resource_profile.contour_id:
|
||||
raise PortableRunDefinitionRegistryError(
|
||||
@@ -720,13 +740,38 @@ class PortableRunDefinitionRegistry:
|
||||
"portable setup and definition identity are not allowlisted"
|
||||
)
|
||||
|
||||
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
|
||||
"""Convert the complete registry and fail if any definition is blocked."""
|
||||
def resolve_setup(self, setup_id: str) -> PortableRunDefinition:
|
||||
"""Resolve the single current definition projected for one setup."""
|
||||
|
||||
return RecordedRunDefinitionRegistry(
|
||||
tuple(definition.to_recorded_run_definition() for definition in self.definitions)
|
||||
_pattern(setup_id, _IDENTIFIER, "setup id")
|
||||
for definition in self.definitions:
|
||||
if definition.setup_id == setup_id:
|
||||
return definition
|
||||
raise PortableRunDefinitionRegistryError("portable setup is not allowlisted")
|
||||
|
||||
def ready_recorded_definitions(self) -> tuple[RecordedRunDefinition, ...]:
|
||||
"""Return only definitions with fully sealed, installed executors.
|
||||
|
||||
Blocked definitions remain valid catalog entries and do not prevent an
|
||||
unrelated ready definition from entering the durable queue allowlist.
|
||||
"""
|
||||
|
||||
return tuple(
|
||||
definition.to_recorded_run_definition()
|
||||
for definition in self.definitions
|
||||
if definition.executor.ready
|
||||
)
|
||||
|
||||
def to_recorded_registry(self) -> RecordedRunDefinitionRegistry:
|
||||
"""Build a queue registry from ready definitions, ignoring blocked ones."""
|
||||
|
||||
definitions = self.ready_recorded_definitions()
|
||||
if not definitions:
|
||||
raise PortableRunDefinitionUnavailableError(
|
||||
"portable registry has no sealed and installed executor"
|
||||
)
|
||||
return RecordedRunDefinitionRegistry(definitions)
|
||||
|
||||
|
||||
def canonical_sha256(value: object) -> str:
|
||||
"""Return the repository-wide canonical JSON SHA-256 identity."""
|
||||
|
||||
@@ -1,15 +1,9 @@
|
||||
"""UI-ready Observatory projection for the portable LAB V1 definition.
|
||||
"""UI-ready Observatory projection for source-independent portable setups.
|
||||
|
||||
This module deliberately does not extend the legacy setup registry and does
|
||||
not submit work. It projects two independent facts for one selected source:
|
||||
|
||||
* the result of a lightweight recorded-source capability probe;
|
||||
* the executor state sealed by the portable RunDefinition;
|
||||
|
||||
Keeping those facts separate prevents a compatible new recording from being
|
||||
described as incompatible merely because the executor is not installed yet.
|
||||
Historical vegetation-shadow results belong only to the legacy catalog and
|
||||
never become exact results of the generic portable v2 definition.
|
||||
The projector keeps source capability, executor availability, and dispatch
|
||||
availability separate. One blocked definition therefore remains visible
|
||||
without hiding another definition or weakening either definition's admission
|
||||
contract. Historical LAB results remain exclusively in the legacy catalog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -17,11 +11,16 @@ from __future__ import annotations
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
from typing import Final, Protocol
|
||||
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
PortableCalculationProfilePolicy,
|
||||
PortableCalculationProfileRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceCapability,
|
||||
@@ -34,6 +33,8 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||
)
|
||||
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||
PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2"
|
||||
PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
|
||||
|
||||
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
@@ -43,6 +44,25 @@ _OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
_SETUP_PRESENTATION: Final = {
|
||||
PORTABLE_LAB_V1_SETUP_ID: {
|
||||
"lab_id": "LAB V1",
|
||||
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
"description": ("Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."),
|
||||
"compatible": "Запись соответствует требованиям EoMT + DDRNet.",
|
||||
"incompatible": "Запись не соответствует требованиям EoMT + DDRNet.",
|
||||
"executor_unavailable": "Вычислительный контур LAB V1 пока недоступен.",
|
||||
},
|
||||
PORTABLE_M49_SETUP_ID: {
|
||||
"lab_id": "LAB M4.9T5",
|
||||
"display_name": PORTABLE_M49_DISPLAY_NAME,
|
||||
"description": ("Динамический TGS-разбор записанной K1-сессии без ML; только наблюдение."),
|
||||
"compatible": "Запись соответствует требованиям TRAVEL TGS.",
|
||||
"incompatible": "Запись не соответствует требованиям TRAVEL TGS.",
|
||||
"executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.",
|
||||
},
|
||||
}
|
||||
|
||||
_MODEL_PRESENTATION: Final = {
|
||||
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
|
||||
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
|
||||
@@ -50,82 +70,104 @@ _MODEL_PRESENTATION: Final = {
|
||||
|
||||
|
||||
class PortableSetupProjectionError(RuntimeError):
|
||||
"""The portable setup cannot be projected without weakening its contract."""
|
||||
"""A portable setup cannot be projected without weakening its contract."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PortableSourceCapabilityProbeService(Protocol):
|
||||
"""A definition-bound lightweight source-capability service."""
|
||||
class PortableDefinitionCapabilityProbe(Protocol):
|
||||
"""Definition-bound lightweight capability probe."""
|
||||
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
"""Probe one source without preparing media or persisting documents."""
|
||||
def probe(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
"""Probe one source against one exact portable definition."""
|
||||
|
||||
|
||||
type PortableSourceCapabilityProbe = (
|
||||
Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService
|
||||
)
|
||||
type PortableSourceCapabilityProbe = Callable[[str], PortableRecordedSourceCapability]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SourceCompatibility:
|
||||
compatible: bool
|
||||
capability: PortableRecordedSourceCapability | None
|
||||
reason: str
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"outcome": "pass" if self.compatible else "blocked",
|
||||
"compatible": self.compatible,
|
||||
"reason": (
|
||||
"Запись соответствует требованиям EoMT + DDRNet."
|
||||
if self.compatible
|
||||
else "Запись не соответствует требованиям EoMT + DDRNet."
|
||||
),
|
||||
"reason": self.reason,
|
||||
}
|
||||
|
||||
|
||||
class PortableLabV1SetupProjector:
|
||||
"""Project the generic portable LAB V1 setup for one selected source."""
|
||||
class PortableSetupProjector:
|
||||
"""Project every allowlisted portable setup for one selected source."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
capability_probe: PortableSourceCapabilityProbe,
|
||||
capability_probe: PortableDefinitionCapabilityProbe,
|
||||
dispatch_available: bool = False,
|
||||
) -> None:
|
||||
self._definition = _resolve_lab_v1_definition(registry)
|
||||
if not isinstance(
|
||||
capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
) and not callable(capability_probe):
|
||||
if not hasattr(capability_probe, "probe"):
|
||||
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
||||
self._registry = registry
|
||||
self._capability_probe = capability_probe
|
||||
_validate_model_presentation(self._definition)
|
||||
if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only")
|
||||
self._dispatch_available = dispatch_available
|
||||
for definition in registry.definitions:
|
||||
_validate_model_presentation(definition)
|
||||
if definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableSetupProjectionError(
|
||||
"portable setup authority is not observation-only"
|
||||
)
|
||||
|
||||
def has_setup(self, setup_id: str) -> bool:
|
||||
try:
|
||||
self._registry.resolve_setup(setup_id)
|
||||
except (PortableRunDefinitionRegistryError, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
def catalog(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a one-setup v2 catalog projection for ``source``."""
|
||||
"""Return all independent portable setup projections for ``source``."""
|
||||
|
||||
return {
|
||||
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
"source_session_id": source.session_id,
|
||||
"setups": [self.project(source)],
|
||||
"setups": [
|
||||
self.project(source, setup_id=definition.setup_id)
|
||||
for definition in self._registry.definitions
|
||||
],
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def project(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a strict, observation-only setup projection."""
|
||||
def project(
|
||||
self,
|
||||
source: SessionSummary,
|
||||
*,
|
||||
setup_id: str,
|
||||
) -> dict[str, object]:
|
||||
"""Return one strict observation-only portable setup projection."""
|
||||
|
||||
_validate_source_id(source.session_id)
|
||||
compatibility = self._probe_source(source.session_id)
|
||||
definition = self._definition
|
||||
try:
|
||||
definition = self._registry.resolve_setup(setup_id)
|
||||
except PortableRunDefinitionRegistryError as exc:
|
||||
raise PortableSetupProjectionError("portable setup is unavailable") from exc
|
||||
presentation = _presentation(definition)
|
||||
compatibility = self._probe_source(definition, source.session_id, presentation)
|
||||
executor = definition.executor
|
||||
submission_allowed = (
|
||||
compatibility.compatible and executor.ready and self._dispatch_available
|
||||
)
|
||||
return {
|
||||
"setup_id": definition.setup_id,
|
||||
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
"description": (
|
||||
"Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."
|
||||
),
|
||||
"display_name": presentation["display_name"],
|
||||
"description": presentation["description"],
|
||||
"origin": "portable-definition",
|
||||
"source_requirements": definition.source_requirements.as_dict(),
|
||||
"run_definition": {
|
||||
@@ -141,65 +183,169 @@ class PortableLabV1SetupProjector:
|
||||
"contour_id": executor.contour_id,
|
||||
"state": executor.state,
|
||||
"ready": executor.ready,
|
||||
"reason_code": executor.reason_code,
|
||||
"reason": executor.reason,
|
||||
},
|
||||
"existing_results": [],
|
||||
"preflight": {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": self._preflight_reason(compatibility),
|
||||
"submission_allowed": False,
|
||||
"outcome": "ready" if submission_allowed else "blocked",
|
||||
"action": "check" if submission_allowed else "blocked",
|
||||
"reason": self._preflight_reason(
|
||||
definition,
|
||||
compatibility,
|
||||
presentation,
|
||||
),
|
||||
"submission_allowed": submission_allowed,
|
||||
"existing_result_ids": [],
|
||||
},
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def _probe_source(self, source_session_id: str) -> _SourceCompatibility:
|
||||
def _probe_source(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
source_session_id: str,
|
||||
presentation: dict[str, str],
|
||||
) -> _SourceCompatibility:
|
||||
try:
|
||||
if isinstance(
|
||||
self._capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
):
|
||||
capability = self._capability_probe.probe(source_session_id)
|
||||
else:
|
||||
capability = self._capability_probe(source_session_id)
|
||||
capability = self._capability_probe.probe(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=definition.setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
except PortableSourceAdmissionError:
|
||||
return _SourceCompatibility(compatible=False, capability=None)
|
||||
return _SourceCompatibility(
|
||||
compatible=False,
|
||||
capability=None,
|
||||
reason=presentation["incompatible"],
|
||||
)
|
||||
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||
raise PortableSetupProjectionError("capability probe returned an invalid result")
|
||||
if capability.source_session_id != source_session_id:
|
||||
raise PortableSetupProjectionError(
|
||||
"capability probe is bound to another source session"
|
||||
)
|
||||
if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256:
|
||||
if capability.source_adapter_sha256 != definition.source_adapter.contract_sha256:
|
||||
raise PortableSetupProjectionError("capability probe uses another source adapter")
|
||||
return _SourceCompatibility(compatible=True, capability=capability)
|
||||
|
||||
def _preflight_reason(self, compatibility: _SourceCompatibility) -> str:
|
||||
if not compatibility.compatible:
|
||||
return "Запись не соответствует требованиям этого сетапа."
|
||||
if not self._definition.executor.ready:
|
||||
return "Вычислительный контур LAB V1 пока недоступен."
|
||||
return (
|
||||
"Server-side проверка definition/check SHA и постановка portable "
|
||||
"LAB V1 в очередь пока недоступны."
|
||||
return _SourceCompatibility(
|
||||
compatible=True,
|
||||
capability=capability,
|
||||
reason=presentation["compatible"],
|
||||
)
|
||||
|
||||
def _preflight_reason(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
compatibility: _SourceCompatibility,
|
||||
presentation: dict[str, str],
|
||||
) -> str:
|
||||
if not compatibility.compatible:
|
||||
return "Запись не соответствует требованиям этого сетапа."
|
||||
if not definition.executor.ready:
|
||||
return presentation["executor_unavailable"]
|
||||
if not self._dispatch_available:
|
||||
return "Server-side проверка и постановка portable-сетапа в очередь недоступны."
|
||||
return "Сетап готов к server-side проверке источника перед постановкой в очередь."
|
||||
|
||||
def _resolve_lab_v1_definition(
|
||||
|
||||
class PortableLabV1SetupProjector(PortableSetupProjector):
|
||||
"""Backward-compatible single-definition LAB V1 projector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
capability_probe: PortableSourceCapabilityProbe | object,
|
||||
) -> None:
|
||||
try:
|
||||
definition = registry.resolve_setup(PORTABLE_LAB_V1_SETUP_ID)
|
||||
except PortableRunDefinitionRegistryError as exc:
|
||||
raise PortableSetupProjectionError(
|
||||
"portable LAB V1 definition is unavailable or ambiguous"
|
||||
) from exc
|
||||
|
||||
class _LegacyProbeAdapter:
|
||||
def probe(
|
||||
adapter_self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
del adapter_self
|
||||
if (
|
||||
setup_id != definition.setup_id
|
||||
or definition_sha256 != definition.definition_sha256
|
||||
):
|
||||
raise PortableSetupProjectionError("portable LAB V1 definition changed")
|
||||
probe_method = getattr(capability_probe, "probe", None)
|
||||
if callable(probe_method):
|
||||
capability = probe_method(source_session_id)
|
||||
elif callable(capability_probe):
|
||||
capability = capability_probe(source_session_id)
|
||||
else:
|
||||
raise PortableSetupProjectionError(
|
||||
"portable source capability probe is unavailable"
|
||||
)
|
||||
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||
raise PortableSetupProjectionError(
|
||||
"capability probe returned an invalid result"
|
||||
)
|
||||
return capability
|
||||
|
||||
super().__init__(
|
||||
registry=PortableRunDefinitionRegistry((definition,)),
|
||||
capability_probe=_LegacyProbeAdapter(),
|
||||
dispatch_available=False,
|
||||
)
|
||||
|
||||
def project(
|
||||
self,
|
||||
source: SessionSummary,
|
||||
*,
|
||||
setup_id: str = PORTABLE_LAB_V1_SETUP_ID,
|
||||
) -> dict[str, object]:
|
||||
return super().project(source, setup_id=setup_id)
|
||||
|
||||
|
||||
def _presentation(definition: PortableRunDefinition) -> dict[str, str]:
|
||||
configured = _SETUP_PRESENTATION.get(definition.setup_id)
|
||||
if configured is not None:
|
||||
return dict(configured)
|
||||
return {
|
||||
"lab_id": "LAB PORTABLE",
|
||||
"display_name": definition.setup_id,
|
||||
"description": "Переносимый анализ записанной K1-сессии; только наблюдение.",
|
||||
"compatible": "Запись соответствует требованиям сетапа.",
|
||||
"incompatible": "Запись не соответствует требованиям сетапа.",
|
||||
"executor_unavailable": "Вычислительный контур сетапа пока недоступен.",
|
||||
}
|
||||
|
||||
|
||||
def portable_calculation_profile_registry(
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
) -> PortableRunDefinition:
|
||||
matching = tuple(
|
||||
definition
|
||||
for definition in registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
if len(matching) != 1:
|
||||
raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous")
|
||||
return matching[0]
|
||||
) -> PortableCalculationProfileRegistry:
|
||||
"""Build exact publication policies from the server-owned presentation map."""
|
||||
|
||||
policies: list[PortableCalculationProfilePolicy] = []
|
||||
for definition in registry.definitions:
|
||||
presentation = _presentation(definition)
|
||||
policies.append(
|
||||
PortableCalculationProfilePolicy(
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
lab_id=presentation["lab_id"],
|
||||
display_name=presentation["display_name"],
|
||||
)
|
||||
)
|
||||
return PortableCalculationProfileRegistry(tuple(policies))
|
||||
|
||||
|
||||
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
||||
if definition.setup_id != PORTABLE_LAB_V1_SETUP_ID:
|
||||
return
|
||||
releases = {model.release_id: model for model in definition.models}
|
||||
if set(releases) != set(_MODEL_PRESENTATION):
|
||||
raise PortableSetupProjectionError(
|
||||
@@ -217,15 +363,14 @@ def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
||||
|
||||
|
||||
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
|
||||
by_release = {model.release_id: model for model in definition.models}
|
||||
return [
|
||||
{
|
||||
"name": _MODEL_PRESENTATION[release_id],
|
||||
"release_id": release_id,
|
||||
"model_id": by_release[release_id].model_id,
|
||||
"architecture": by_release[release_id].architecture,
|
||||
"name": _MODEL_PRESENTATION.get(model.release_id, model.model_id),
|
||||
"release_id": model.release_id,
|
||||
"model_id": model.model_id,
|
||||
"architecture": model.architecture,
|
||||
}
|
||||
for release_id in _MODEL_PRESENTATION
|
||||
for model in definition.models
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
"""Fail-closed server composition for portable Observatory Worker jobs.
|
||||
|
||||
This module is deliberately only a composition boundary. It does not enable
|
||||
the Worker router, install an executor, select commands, or grant production
|
||||
authority. It binds the two admitted portable profiles to their exact result
|
||||
contracts, then constructs the local artifact transport and verified result
|
||||
publisher from server-owned dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
M49_PORTABLE_RESULT_SCHEMA,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_executor import (
|
||||
PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||
validate_lab_v1_result_v2,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PortableCalculationProfileRegistry,
|
||||
PortableResultContractValidator,
|
||||
PortableResultContractValidatorRegistration,
|
||||
PortableResultContractValidatorRegistry,
|
||||
PortableResultPublicationBlockedError,
|
||||
)
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_SETUP_ID,
|
||||
portable_calculation_profile_registry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||
from k1link.sessions.media import RecordedMediaInspector
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256: Final = (
|
||||
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
|
||||
)
|
||||
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CAS_ROOT"
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
|
||||
)
|
||||
|
||||
|
||||
class PortableWorkerIntegrationError(RuntimeError):
|
||||
"""The exact portable server composition cannot be constructed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerStorageRoots:
|
||||
"""Pre-provisioned server-owned roots for large portable artifacts."""
|
||||
|
||||
source_cas_root: Path
|
||||
result_staging_root: Path
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
*,
|
||||
artifact_store_root: Path,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> PortableWorkerStorageRoots:
|
||||
"""Load both roots without creating anything on a missing mount."""
|
||||
|
||||
values = os.environ if environment is None else environment
|
||||
return cls.from_paths(
|
||||
artifact_store_root=artifact_store_root,
|
||||
source_cas_root=_required_environment_path(
|
||||
values,
|
||||
OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV,
|
||||
),
|
||||
result_staging_root=_required_environment_path(
|
||||
values,
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV,
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_paths(
|
||||
cls,
|
||||
*,
|
||||
artifact_store_root: Path,
|
||||
source_cas_root: Path,
|
||||
result_staging_root: Path,
|
||||
) -> PortableWorkerStorageRoots:
|
||||
"""Validate existing, disjoint roots inside the central store boundary."""
|
||||
|
||||
artifact_store = _existing_canonical_directory(
|
||||
artifact_store_root,
|
||||
"central artifact store",
|
||||
)
|
||||
storage_boundary = _existing_canonical_directory(
|
||||
artifact_store.parent,
|
||||
"central artifact storage boundary",
|
||||
)
|
||||
_require_mounted_volume(storage_boundary)
|
||||
source_cas = _existing_canonical_directory(
|
||||
source_cas_root,
|
||||
"portable source CAS",
|
||||
)
|
||||
result_staging = _existing_canonical_directory(
|
||||
result_staging_root,
|
||||
"portable result staging",
|
||||
)
|
||||
for label, root in (
|
||||
("portable source CAS", source_cas),
|
||||
("portable result staging", result_staging),
|
||||
):
|
||||
if root == storage_boundary or not root.is_relative_to(storage_boundary):
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"{label} must be inside the central artifact storage boundary"
|
||||
)
|
||||
if _paths_overlap(root, artifact_store):
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"{label} must be disjoint from the central artifact store"
|
||||
)
|
||||
if _paths_overlap(source_cas, result_staging):
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable source CAS and result staging roots must be disjoint"
|
||||
)
|
||||
return cls(
|
||||
source_cas_root=source_cas,
|
||||
result_staging_root=result_staging,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableObservatoryWorkerIntegration:
|
||||
"""Server-owned objects required by the optional Worker router."""
|
||||
|
||||
calculation_profiles: PortableCalculationProfileRegistry
|
||||
validators: PortableResultContractValidatorRegistry
|
||||
artifact_transport: PortableObservatoryArtifactTransport
|
||||
result_publisher: PortableObservatoryResultPublisher
|
||||
supported_setup_ids: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatorSpec:
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
contract_id: str
|
||||
contract_version: int
|
||||
result_schema: str
|
||||
result_kind: str
|
||||
contract_sha256: str
|
||||
validator: PortableResultContractValidator
|
||||
|
||||
def expected_contract(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||
"contract_id": self.contract_id,
|
||||
"version": self.contract_version,
|
||||
"result_schema": self.result_schema,
|
||||
"result_kind": self.result_kind,
|
||||
"publication": "observatory",
|
||||
"contract_sha256": self.contract_sha256,
|
||||
}
|
||||
|
||||
|
||||
_VALIDATOR_SPECS: Final = (
|
||||
_ValidatorSpec(
|
||||
setup_id=PORTABLE_LAB_V1_SETUP_ID,
|
||||
definition_id="lab-v1-eomt-ddrnet-portable",
|
||||
definition_version=2,
|
||||
contract_id="recorded-eomt-ddrnet-review-v2",
|
||||
contract_version=2,
|
||||
result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||
result_kind="recorded-perception-qualification",
|
||||
contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_lab_v1_result_v2,
|
||||
),
|
||||
_ValidatorSpec(
|
||||
setup_id=PORTABLE_M49_SETUP_ID,
|
||||
definition_id="m49-tgs-portable",
|
||||
definition_version=2,
|
||||
contract_id="m49-tgs-portable-review-v2",
|
||||
contract_version=2,
|
||||
result_schema=M49_PORTABLE_RESULT_SCHEMA,
|
||||
result_kind="recorded-perception-qualification",
|
||||
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_m49_portable_result,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def portable_result_validator_registry(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
) -> PortableResultContractValidatorRegistry:
|
||||
"""Bind both product profiles to fixed result contracts and validators."""
|
||||
|
||||
registrations: list[PortableResultContractValidatorRegistration] = []
|
||||
for spec in _VALIDATOR_SPECS:
|
||||
try:
|
||||
definition = definitions.resolve_setup(spec.setup_id)
|
||||
except PortableRunDefinitionRegistryError as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"required portable setup is unavailable: {spec.setup_id}"
|
||||
) from exc
|
||||
_verify_validator_definition(definition, spec)
|
||||
registrations.append(
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=spec.contract_sha256,
|
||||
validator=spec.validator,
|
||||
)
|
||||
)
|
||||
return PortableResultContractValidatorRegistry(tuple(registrations))
|
||||
|
||||
|
||||
def build_portable_observatory_worker_integration(
|
||||
*,
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
session_store: SessionStore,
|
||||
media_inspector: RecordedMediaInspector,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
artifact_store: CentralArtifactStore,
|
||||
calculation_profiles: PortableCalculationProfileRegistry | None = None,
|
||||
validators: PortableResultContractValidatorRegistry | None = None,
|
||||
source_cas_root: Path | None = None,
|
||||
result_staging_root: Path | None = None,
|
||||
) -> PortableObservatoryWorkerIntegration:
|
||||
"""Construct the dormant server foundation without enabling any route."""
|
||||
|
||||
try:
|
||||
if source_cas_root is None or result_staging_root is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable Worker storage roots must be explicitly configured"
|
||||
)
|
||||
storage_roots = PortableWorkerStorageRoots.from_paths(
|
||||
artifact_store_root=artifact_store.root,
|
||||
source_cas_root=source_cas_root,
|
||||
result_staging_root=result_staging_root,
|
||||
)
|
||||
profiles = calculation_profiles or portable_calculation_profile_registry(definitions)
|
||||
validator_registry = validators or portable_result_validator_registry(definitions)
|
||||
_verify_composition(definitions, profiles, validator_registry)
|
||||
transport = PortableObservatoryArtifactTransport(
|
||||
queue=queue,
|
||||
session_store=session_store,
|
||||
media_inspector=media_inspector,
|
||||
definitions=definitions,
|
||||
source_cas_root=storage_roots.source_cas_root,
|
||||
result_staging_root=storage_roots.result_staging_root,
|
||||
)
|
||||
except PortableWorkerIntegrationError:
|
||||
raise
|
||||
except (
|
||||
PortableArtifactTransportError,
|
||||
PortableResultPublicationBlockedError,
|
||||
PortableRunDefinitionRegistryError,
|
||||
OSError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable Worker server composition is unavailable"
|
||||
) from exc
|
||||
publisher = PortableObservatoryResultPublisher(
|
||||
session_store=session_store,
|
||||
artifact_store=artifact_store,
|
||||
definitions=definitions,
|
||||
calculation_profiles=profiles,
|
||||
validators=validator_registry,
|
||||
)
|
||||
return PortableObservatoryWorkerIntegration(
|
||||
calculation_profiles=profiles,
|
||||
validators=validator_registry,
|
||||
artifact_transport=transport,
|
||||
result_publisher=publisher,
|
||||
supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS),
|
||||
)
|
||||
|
||||
|
||||
def _verify_validator_definition(
|
||||
definition: PortableRunDefinition,
|
||||
spec: _ValidatorSpec,
|
||||
) -> None:
|
||||
if (
|
||||
definition.definition_id != spec.definition_id
|
||||
or definition.version != spec.definition_version
|
||||
or definition.result_contract.as_dict() != spec.expected_contract()
|
||||
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"portable result contract changed for setup: {spec.setup_id}"
|
||||
)
|
||||
|
||||
|
||||
def _verify_composition(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
profiles: PortableCalculationProfileRegistry,
|
||||
validators: PortableResultContractValidatorRegistry,
|
||||
) -> None:
|
||||
expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS}
|
||||
for definition in definitions.definitions:
|
||||
profiles.resolve(definition)
|
||||
if definition.executor.ready:
|
||||
try:
|
||||
validators.resolve(definition.result_contract.contract_sha256)
|
||||
except PortableResultPublicationBlockedError as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"a ready portable definition has no exact result validator"
|
||||
) from exc
|
||||
for contract_sha256, expected in expected_validators.items():
|
||||
if validators.resolve(contract_sha256) is not expected:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable result validator registration changed identity"
|
||||
)
|
||||
|
||||
|
||||
def _required_environment_path(
|
||||
environment: Mapping[str, str],
|
||||
name: str,
|
||||
) -> Path:
|
||||
raw = environment.get(name)
|
||||
if not isinstance(raw, str) or not raw or raw != raw.strip():
|
||||
raise PortableWorkerIntegrationError(f"{name} is required")
|
||||
path = Path(raw)
|
||||
if not path.is_absolute():
|
||||
raise PortableWorkerIntegrationError(f"{name} must be an absolute path")
|
||||
return path
|
||||
|
||||
|
||||
def _existing_canonical_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise PortableWorkerIntegrationError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISDIR(metadata.st_mode)
|
||||
or resolved != candidate
|
||||
):
|
||||
raise PortableWorkerIntegrationError(f"{label} is not a canonical directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _paths_overlap(left: Path, right: Path) -> bool:
|
||||
return left == right or left.is_relative_to(right) or right.is_relative_to(left)
|
||||
|
||||
|
||||
def _require_mounted_volume(path: Path) -> None:
|
||||
parts = path.parts
|
||||
if len(parts) < 3 or parts[0] != os.sep or parts[1] != "Volumes":
|
||||
return
|
||||
mount_point = Path(os.sep, "Volumes", parts[2])
|
||||
if not os.path.ismount(mount_point):
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"central artifact volume is not mounted: {mount_point}"
|
||||
)
|
||||
@@ -0,0 +1,969 @@
|
||||
"""Fail-closed local runtime contract for portable Observatory executors.
|
||||
|
||||
This module belongs on Worker 006, not in the browser or the K1 control path.
|
||||
It binds one source-independent portable RunDefinition to locally verified
|
||||
assets and to local Python adapters. A queued job can select an adapter only
|
||||
through the four server-sealed executor digests; it can never supply a command,
|
||||
path, environment variable, image reference, or priority.
|
||||
|
||||
The production candidate registry is deliberately allowed to contain blocked
|
||||
candidates. Reusable historical assets are evidence, not an installed
|
||||
executor. A blocked candidate cannot yield a Worker registration or execute a
|
||||
job even when every reusable asset is present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
|
||||
PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-worker-runtime-registry/v1"
|
||||
)
|
||||
PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-worker-runtime-candidate/v1"
|
||||
)
|
||||
PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-worker-runtime-plan/v1"
|
||||
)
|
||||
|
||||
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
_SESSION_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_JOB_ID: Final = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
type CandidateState = Literal["blocked", "ready"]
|
||||
type PhaseState = Literal["implemented", "missing"]
|
||||
type RuntimeAssetKind = Literal[
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"model-artifact",
|
||||
]
|
||||
type AssetVerificationState = Literal["matched", "missing", "mismatched"]
|
||||
|
||||
|
||||
class PortableWorkerRuntimeError(RuntimeError):
|
||||
"""Base error for the local portable Worker runtime boundary."""
|
||||
|
||||
|
||||
class PortableWorkerRuntimeRegistryError(PortableWorkerRuntimeError):
|
||||
"""A candidate registry is malformed or drifts from a RunDefinition."""
|
||||
|
||||
|
||||
class PortableWorkerRuntimeUnavailableError(PortableWorkerRuntimeError):
|
||||
"""A candidate is not a complete, sealed, locally admitted executor."""
|
||||
|
||||
|
||||
class PortableWorkerRuntimeJobRejectedError(PortableWorkerRuntimeError):
|
||||
"""A Worker job differs from the exact local candidate identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimePhase:
|
||||
phase_id: str
|
||||
state: PhaseState
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.phase_id, _IDENTIFIER, "runtime phase id")
|
||||
if self.state not in ("implemented", "missing"):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {"phase_id": self.phase_id, "state": self.state}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerAssetRequirement:
|
||||
"""One exact reusable local asset; its locator is intentionally absent."""
|
||||
|
||||
asset_id: str
|
||||
kind: RuntimeAssetKind
|
||||
sha256: str
|
||||
byte_length: int | None
|
||||
component_id: str | None
|
||||
model_release_id: str | None
|
||||
model_artifact_role: str | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.asset_id, _ASSET_ID, "runtime asset id")
|
||||
if self.kind not in (
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"model-artifact",
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||
_digest(self.sha256, "runtime asset sha256")
|
||||
if self.byte_length is not None and (
|
||||
isinstance(self.byte_length, bool)
|
||||
or not isinstance(self.byte_length, int)
|
||||
or self.byte_length < 1
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime asset byte length is invalid")
|
||||
if self.kind == "container-image":
|
||||
if any(
|
||||
value is not None
|
||||
for value in (
|
||||
self.byte_length,
|
||||
self.component_id,
|
||||
self.model_release_id,
|
||||
self.model_artifact_role,
|
||||
)
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"container image requirement cannot impersonate a definition asset"
|
||||
)
|
||||
elif self.kind == "definition-component":
|
||||
_optional_identifier(self.component_id, "definition component id")
|
||||
if self.component_id is None or any(
|
||||
value is not None
|
||||
for value in (self.model_release_id, self.model_artifact_role)
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"definition component requirement is incomplete"
|
||||
)
|
||||
elif self.kind == "model-artifact":
|
||||
_optional_identifier(self.model_release_id, "model release id")
|
||||
_optional_identifier(self.model_artifact_role, "model artifact role")
|
||||
if (
|
||||
self.model_release_id is None
|
||||
or self.model_artifact_role is None
|
||||
or self.component_id is not None
|
||||
or self.byte_length is None
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"model artifact requirement is incomplete"
|
||||
)
|
||||
elif any(
|
||||
value is not None
|
||||
for value in (
|
||||
self.component_id,
|
||||
self.model_release_id,
|
||||
self.model_artifact_role,
|
||||
)
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"local file requirement cannot impersonate a definition asset"
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"asset_id": self.asset_id,
|
||||
"kind": self.kind,
|
||||
"sha256": self.sha256,
|
||||
"byte_length": self.byte_length,
|
||||
"component_id": self.component_id,
|
||||
"model_release_id": self.model_release_id,
|
||||
"model_artifact_role": self.model_artifact_role,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerExecutorSeal:
|
||||
release_id: str
|
||||
release_sha256: str
|
||||
image_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.release_id, _IDENTIFIER, "executor release id")
|
||||
_digest(self.release_sha256, "executor release sha256")
|
||||
_digest(self.image_sha256, "executor image sha256")
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"release_id": self.release_id,
|
||||
"release_sha256": self.release_sha256,
|
||||
"image_sha256": self.image_sha256,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimeCandidate:
|
||||
adapter_id: str
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
source_adapter_sha256: str
|
||||
model_manifest_sha256: str
|
||||
resource_profile_sha256: str
|
||||
result_contract_sha256: str
|
||||
state: CandidateState
|
||||
executor: PortableWorkerExecutorSeal | None
|
||||
reusable_assets: tuple[PortableWorkerAssetRequirement, ...]
|
||||
phases: tuple[PortableWorkerRuntimePhase, ...]
|
||||
blockers: tuple[str, ...]
|
||||
candidate_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value, label in (
|
||||
(self.adapter_id, "runtime adapter id"),
|
||||
(self.setup_id, "setup id"),
|
||||
(self.definition_id, "definition id"),
|
||||
):
|
||||
_pattern(value, _IDENTIFIER, label)
|
||||
if (
|
||||
isinstance(self.definition_version, bool)
|
||||
or not isinstance(self.definition_version, int)
|
||||
or self.definition_version < 1
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("definition version is invalid")
|
||||
for value, label in (
|
||||
(self.definition_sha256, "definition sha256"),
|
||||
(self.source_adapter_sha256, "source adapter sha256"),
|
||||
(self.model_manifest_sha256, "model manifest sha256"),
|
||||
(self.resource_profile_sha256, "resource profile sha256"),
|
||||
(self.result_contract_sha256, "result contract sha256"),
|
||||
(self.candidate_sha256, "runtime candidate sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
if self.state not in ("blocked", "ready"):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
|
||||
asset_ids = [asset.asset_id for asset in self.reusable_assets]
|
||||
if asset_ids != sorted(asset_ids) or len(asset_ids) != len(set(asset_ids)):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime assets must be unique and canonically ordered"
|
||||
)
|
||||
phase_ids = [phase.phase_id for phase in self.phases]
|
||||
if not phase_ids or len(phase_ids) != len(set(phase_ids)):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime phases must be non-empty and unique")
|
||||
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
|
||||
set(self.blockers)
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime blockers must be unique and canonically ordered"
|
||||
)
|
||||
for blocker in self.blockers:
|
||||
_pattern(blocker, _IDENTIFIER, "runtime blocker")
|
||||
missing_phases = tuple(
|
||||
phase.phase_id for phase in self.phases if phase.state == "missing"
|
||||
)
|
||||
if self.state == "ready":
|
||||
if self.executor is None or self.blockers or missing_phases:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"ready runtime requires a sealed executor and complete phases"
|
||||
)
|
||||
elif self.executor is not None or not self.blockers or not missing_phases:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"blocked runtime must keep its executor unsealed and missing phases explicit"
|
||||
)
|
||||
if self.candidate_sha256 != canonical_sha256(self.identity_document()):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime candidate digest changed")
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self.state == "ready"
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA,
|
||||
"adapter_id": self.adapter_id,
|
||||
"setup_id": self.setup_id,
|
||||
"definition_id": self.definition_id,
|
||||
"definition_version": self.definition_version,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"model_manifest_sha256": self.model_manifest_sha256,
|
||||
"resource_profile_sha256": self.resource_profile_sha256,
|
||||
"result_contract_sha256": self.result_contract_sha256,
|
||||
"state": self.state,
|
||||
"executor": self.executor.as_dict() if self.executor is not None else None,
|
||||
"reusable_assets": [asset.as_dict() for asset in self.reusable_assets],
|
||||
"phases": [phase.as_dict() for phase in self.phases],
|
||||
"blockers": list(self.blockers),
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
def bind_definition(self, definition: PortableRunDefinition) -> None:
|
||||
if (
|
||||
definition.setup_id != self.setup_id
|
||||
or definition.definition_id != self.definition_id
|
||||
or definition.version != self.definition_version
|
||||
or definition.definition_sha256 != self.definition_sha256
|
||||
or definition.source_adapter.contract_sha256 != self.source_adapter_sha256
|
||||
or definition.model_manifest_sha256 != self.model_manifest_sha256
|
||||
or definition.resource_profile.profile_sha256 != self.resource_profile_sha256
|
||||
or definition.result_contract.contract_sha256 != self.result_contract_sha256
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime candidate and portable RunDefinition identities disagree"
|
||||
)
|
||||
components = {component.component_id: component for component in definition.components}
|
||||
models = {model.release_id: model for model in definition.models}
|
||||
for requirement in self.reusable_assets:
|
||||
if requirement.kind == "definition-component":
|
||||
component = components.get(requirement.component_id or "")
|
||||
if component is None or component.sha256 != requirement.sha256:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime component requirement differs from its RunDefinition"
|
||||
)
|
||||
elif requirement.kind == "model-artifact":
|
||||
model = models.get(requirement.model_release_id or "")
|
||||
artifacts = {
|
||||
artifact.role: artifact for artifact in model.artifacts
|
||||
} if model is not None else {}
|
||||
artifact = artifacts.get(requirement.model_artifact_role or "")
|
||||
if (
|
||||
artifact is None
|
||||
or artifact.sha256 != requirement.sha256
|
||||
or artifact.byte_length != requirement.byte_length
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime model artifact differs from its RunDefinition"
|
||||
)
|
||||
if self.state == "blocked":
|
||||
if definition.executor.ready:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"blocked local runtime cannot bind a ready RunDefinition"
|
||||
)
|
||||
return
|
||||
if not definition.executor.ready or self.executor is None:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"ready local runtime requires a ready RunDefinition"
|
||||
)
|
||||
if (
|
||||
definition.executor.release_id != self.executor.release_id
|
||||
or definition.executor.release_sha256 != self.executor.release_sha256
|
||||
or definition.executor.image_sha256 != self.executor.image_sha256
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"local executor seal differs from its RunDefinition"
|
||||
)
|
||||
|
||||
def executor_identity(self) -> ObservatoryWorkerExecutorIdentity:
|
||||
if not self.ready or self.executor is None:
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"blocked runtime candidate has no executor identity"
|
||||
)
|
||||
return ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=self.executor.release_sha256,
|
||||
image_sha256=self.executor.image_sha256,
|
||||
model_manifest_sha256=self.model_manifest_sha256,
|
||||
resource_profile_sha256=self.resource_profile_sha256,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimeRegistry:
|
||||
candidates: tuple[PortableWorkerRuntimeCandidate, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.candidates:
|
||||
raise PortableWorkerRuntimeRegistryError("runtime registry cannot be empty")
|
||||
for label, values in (
|
||||
("runtime adapter ids", [candidate.adapter_id for candidate in self.candidates]),
|
||||
("runtime setup ids", [candidate.setup_id for candidate in self.candidates]),
|
||||
(
|
||||
"runtime candidate digests",
|
||||
[candidate.candidate_sha256 for candidate in self.candidates],
|
||||
),
|
||||
):
|
||||
if len(values) != len(set(values)):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} must be unique")
|
||||
|
||||
@classmethod
|
||||
def from_file(
|
||||
cls,
|
||||
path: Path,
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
) -> PortableWorkerRuntimeRegistry:
|
||||
candidate_path = path.expanduser().absolute()
|
||||
if candidate_path.is_symlink() or not candidate_path.is_file():
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime registry must be a regular file"
|
||||
)
|
||||
try:
|
||||
if candidate_path.stat().st_size > _MAX_REGISTRY_BYTES:
|
||||
raise PortableWorkerRuntimeRegistryError("runtime registry is too large")
|
||||
document: object = json.loads(candidate_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableWorkerRuntimeRegistryError("runtime registry is unreadable") from exc
|
||||
_reject_unsafe_keys(document)
|
||||
root = _object(document, "runtime registry")
|
||||
_exact_keys(root, {"schema_version", "candidates"}, "runtime registry")
|
||||
if root["schema_version"] != PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA:
|
||||
raise PortableWorkerRuntimeRegistryError("runtime registry schema is invalid")
|
||||
rows = _array(root["candidates"], "runtime candidates")
|
||||
registry = cls(tuple(_candidate(row) for row in rows))
|
||||
for candidate in registry.candidates:
|
||||
candidate.bind_definition(
|
||||
definitions.resolve(candidate.setup_id, candidate.definition_sha256)
|
||||
)
|
||||
return registry
|
||||
|
||||
def resolve(self, setup_id: str, definition_sha256: str) -> PortableWorkerRuntimeCandidate:
|
||||
_pattern(setup_id, _IDENTIFIER, "setup id")
|
||||
_digest(definition_sha256, "definition sha256")
|
||||
for candidate in self.candidates:
|
||||
if (
|
||||
candidate.setup_id == setup_id
|
||||
and candidate.definition_sha256 == definition_sha256
|
||||
):
|
||||
return candidate
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"portable Worker candidate identity is not allowlisted"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerLocalAssetBinding:
|
||||
"""Worker-local binding populated by reviewed local configuration only."""
|
||||
|
||||
asset_id: str
|
||||
file_path: Path | None = None
|
||||
image_sha256: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.asset_id, _ASSET_ID, "local asset id")
|
||||
if (self.file_path is None) == (self.image_sha256 is None):
|
||||
raise ValueError("local asset binding must select exactly one local locator")
|
||||
if self.image_sha256 is not None:
|
||||
_digest(self.image_sha256, "local image sha256")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerAssetVerification:
|
||||
asset_id: str
|
||||
state: AssetVerificationState
|
||||
reason: str | None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.asset_id, _ASSET_ID, "verified asset id")
|
||||
if self.state not in ("matched", "missing", "mismatched"):
|
||||
raise ValueError("verified asset state is invalid")
|
||||
if (self.state == "matched") != (self.reason is None):
|
||||
raise ValueError("verified asset reason disagrees with its state")
|
||||
|
||||
|
||||
def verify_local_assets(
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
|
||||
) -> tuple[PortableWorkerAssetVerification, ...]:
|
||||
"""Hash locally bound assets without accepting locators from a job."""
|
||||
|
||||
checks: list[PortableWorkerAssetVerification] = []
|
||||
for requirement in candidate.reusable_assets:
|
||||
binding = bindings.get(requirement.asset_id)
|
||||
if binding is None or binding.asset_id != requirement.asset_id:
|
||||
checks.append(
|
||||
PortableWorkerAssetVerification(requirement.asset_id, "missing", "not-bound")
|
||||
)
|
||||
continue
|
||||
if requirement.kind == "container-image":
|
||||
matched = binding.file_path is None and binding.image_sha256 == requirement.sha256
|
||||
else:
|
||||
matched = _matches_file(requirement, binding.file_path)
|
||||
checks.append(
|
||||
PortableWorkerAssetVerification(
|
||||
requirement.asset_id,
|
||||
"matched" if matched else "mismatched",
|
||||
None if matched else "identity-mismatch",
|
||||
)
|
||||
)
|
||||
return tuple(checks)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimeAdmission:
|
||||
candidate_sha256: str
|
||||
ready: bool
|
||||
blockers: tuple[str, ...]
|
||||
assets: tuple[PortableWorkerAssetVerification, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_digest(self.candidate_sha256, "runtime admission candidate sha256")
|
||||
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
|
||||
set(self.blockers)
|
||||
):
|
||||
raise ValueError("runtime admission blockers must be canonical")
|
||||
if self.ready and (
|
||||
self.blockers or any(item.state != "matched" for item in self.assets)
|
||||
):
|
||||
raise ValueError("ready runtime admission cannot contain an unresolved asset")
|
||||
|
||||
|
||||
def inspect_runtime_candidate(
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
bindings: Mapping[str, PortableWorkerLocalAssetBinding],
|
||||
) -> PortableWorkerRuntimeAdmission:
|
||||
assets = verify_local_assets(candidate, bindings)
|
||||
asset_blockers = tuple(
|
||||
sorted(f"asset-{item.asset_id}-{item.state}" for item in assets if item.state != "matched")
|
||||
)
|
||||
blockers = tuple(sorted((*candidate.blockers, *asset_blockers)))
|
||||
return PortableWorkerRuntimeAdmission(
|
||||
candidate_sha256=candidate.candidate_sha256,
|
||||
ready=candidate.ready and not blockers,
|
||||
blockers=blockers,
|
||||
assets=assets,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerSourceStage:
|
||||
"""A locally materialized source; transport/materialization owns its path."""
|
||||
|
||||
root: Path
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value, label in (
|
||||
(self.source_bundle_sha256, "source bundle sha256"),
|
||||
(self.source_capability_manifest_sha256, "source capability sha256"),
|
||||
(self.source_adapter_sha256, "source adapter sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
if self.root.is_symlink() or not self.root.is_dir():
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"portable source stage must be a real local directory"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerResultDraft:
|
||||
root: Path
|
||||
result_id: str
|
||||
result_sha256: str
|
||||
result_contract_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.result_id, _SESSION_ID, "result id")
|
||||
_digest(self.result_sha256, "result sha256")
|
||||
_digest(self.result_contract_sha256, "result contract sha256")
|
||||
if self.root.is_symlink() or not self.root.is_dir():
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"portable result draft must be a real local directory"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimePlan:
|
||||
job_id: str
|
||||
adapter_id: str
|
||||
candidate_sha256: str
|
||||
setup_id: str
|
||||
definition_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
result_contract_sha256: str
|
||||
phases: tuple[str, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.job_id, _JOB_ID, "runtime plan job id")
|
||||
for value, label in (
|
||||
(self.adapter_id, "runtime plan adapter id"),
|
||||
(self.setup_id, "runtime plan setup id"),
|
||||
):
|
||||
_pattern(value, _IDENTIFIER, label)
|
||||
for value, label in (
|
||||
(self.candidate_sha256, "runtime plan candidate sha256"),
|
||||
(self.definition_sha256, "runtime plan definition sha256"),
|
||||
(self.source_bundle_sha256, "runtime plan source bundle sha256"),
|
||||
(
|
||||
self.source_capability_manifest_sha256,
|
||||
"runtime plan source capability sha256",
|
||||
),
|
||||
(self.result_contract_sha256, "runtime plan result contract sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
if not self.phases or len(self.phases) != len(set(self.phases)):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime plan phases must be non-empty and unique"
|
||||
)
|
||||
for phase in self.phases:
|
||||
_pattern(phase, _IDENTIFIER, "runtime plan phase id")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA,
|
||||
"job_id": self.job_id,
|
||||
"adapter_id": self.adapter_id,
|
||||
"candidate_sha256": self.candidate_sha256,
|
||||
"setup_id": self.setup_id,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": self.source_capability_manifest_sha256,
|
||||
"result_contract_sha256": self.result_contract_sha256,
|
||||
"phases": list(self.phases),
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
class PortableWorkerSourceMaterializer(Protocol):
|
||||
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage: ...
|
||||
|
||||
|
||||
class PortableWorkerProfileRunner(Protocol):
|
||||
def run(
|
||||
self,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
source: PortableWorkerSourceStage,
|
||||
) -> PortableWorkerResultDraft: ...
|
||||
|
||||
|
||||
class PortableWorkerResultPublisher(Protocol):
|
||||
def publish(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
draft: PortableWorkerResultDraft,
|
||||
) -> ObservatoryWorkerExecutionResult: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerExecutorAdapter:
|
||||
"""Local adapter composition; none of its dependencies come from a job."""
|
||||
|
||||
candidate: PortableWorkerRuntimeCandidate
|
||||
definition: PortableRunDefinition
|
||||
admission: PortableWorkerRuntimeAdmission
|
||||
source_materializer: PortableWorkerSourceMaterializer
|
||||
runner: PortableWorkerProfileRunner
|
||||
publisher: PortableWorkerResultPublisher
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.candidate.bind_definition(self.definition)
|
||||
if not self.candidate.ready or not self.admission.ready:
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"portable Worker adapter cannot bind a blocked runtime candidate"
|
||||
)
|
||||
if self.admission.candidate_sha256 != self.candidate.candidate_sha256:
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"runtime admission belongs to another candidate"
|
||||
)
|
||||
expected_assets = tuple(
|
||||
requirement.asset_id for requirement in self.candidate.reusable_assets
|
||||
)
|
||||
admitted_assets = tuple(item.asset_id for item in self.admission.assets)
|
||||
if admitted_assets != expected_assets or any(
|
||||
item.state != "matched" for item in self.admission.assets
|
||||
):
|
||||
raise PortableWorkerRuntimeUnavailableError(
|
||||
"runtime admission does not prove every candidate asset"
|
||||
)
|
||||
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
self._verify_job(job)
|
||||
source = self.source_materializer.materialize(job)
|
||||
if (
|
||||
source.source_bundle_sha256 != job.source_bundle_sha256
|
||||
or source.source_capability_manifest_sha256
|
||||
!= job.source_capability_manifest_sha256
|
||||
or source.source_adapter_sha256 != job.source_adapter_sha256
|
||||
):
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"materialized source differs from the sealed job"
|
||||
)
|
||||
plan = PortableWorkerRuntimePlan(
|
||||
job_id=job.job_id,
|
||||
adapter_id=self.candidate.adapter_id,
|
||||
candidate_sha256=self.candidate.candidate_sha256,
|
||||
setup_id=job.setup_id,
|
||||
definition_sha256=job.definition_sha256,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||
)
|
||||
draft = self.runner.run(plan, source)
|
||||
if draft.result_contract_sha256 != self.candidate.result_contract_sha256:
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"runtime result uses another result contract"
|
||||
)
|
||||
published = self.publisher.publish(job, draft)
|
||||
if (
|
||||
published.result_id != draft.result_id
|
||||
or published.result_sha256 != draft.result_sha256
|
||||
):
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"publisher receipt differs from the validated result draft"
|
||||
)
|
||||
return published
|
||||
|
||||
def _verify_job(self, job: SealedObservatoryRecordedJob) -> None:
|
||||
executor = self.candidate.executor
|
||||
expected_identity = self.candidate.executor_identity()
|
||||
if (
|
||||
executor is None
|
||||
or job.setup_id != self.definition.setup_id
|
||||
or job.definition_id != self.definition.definition_id
|
||||
or job.definition_version != self.definition.version
|
||||
or job.definition_sha256 != self.definition.definition_sha256
|
||||
or job.source_adapter_id != self.definition.source_adapter.adapter_id
|
||||
or job.source_adapter_version != self.definition.source_adapter.version
|
||||
or job.source_adapter_sha256 != self.definition.source_adapter.contract_sha256
|
||||
or job.executor_release_id != executor.release_id
|
||||
or job.executor_identity != expected_identity
|
||||
or job.model_release_ids != self.definition.learned_models
|
||||
or job.resource_profile_id != self.definition.resource_profile.profile_id
|
||||
or job.checkpoint_policy != self.definition.resource_profile.checkpoint_policy
|
||||
or job.allowed_checkpoints != self.definition.resource_profile.allowed_checkpoints
|
||||
):
|
||||
raise PortableWorkerRuntimeJobRejectedError(
|
||||
"Worker job differs from the exact local runtime identity"
|
||||
)
|
||||
|
||||
|
||||
def _matches_file(
|
||||
requirement: PortableWorkerAssetRequirement,
|
||||
path: Path | None,
|
||||
) -> bool:
|
||||
if path is None:
|
||||
return False
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
if candidate.is_symlink() or not candidate.is_file():
|
||||
return False
|
||||
if requirement.byte_length is not None and candidate.stat().st_size != (
|
||||
requirement.byte_length
|
||||
):
|
||||
return False
|
||||
digest = hashlib.sha256()
|
||||
with candidate.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest() == requirement.sha256
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _candidate(value: object) -> PortableWorkerRuntimeCandidate:
|
||||
row = _object(value, "runtime candidate")
|
||||
_exact_keys(
|
||||
row,
|
||||
{
|
||||
"schema_version",
|
||||
"adapter_id",
|
||||
"setup_id",
|
||||
"definition_id",
|
||||
"definition_version",
|
||||
"definition_sha256",
|
||||
"source_adapter_sha256",
|
||||
"model_manifest_sha256",
|
||||
"resource_profile_sha256",
|
||||
"result_contract_sha256",
|
||||
"state",
|
||||
"executor",
|
||||
"reusable_assets",
|
||||
"phases",
|
||||
"blockers",
|
||||
"authority",
|
||||
"candidate_sha256",
|
||||
},
|
||||
"runtime candidate",
|
||||
)
|
||||
if row["schema_version"] != PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA:
|
||||
raise PortableWorkerRuntimeRegistryError("runtime candidate schema is invalid")
|
||||
if row["authority"] != _AUTHORITY:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime candidate authority must remain observation-only"
|
||||
)
|
||||
state = row["state"]
|
||||
if state not in ("blocked", "ready"):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime candidate state is invalid")
|
||||
executor_row = row["executor"]
|
||||
executor = None if executor_row is None else _executor(executor_row)
|
||||
assets = _array(row["reusable_assets"], "runtime reusable assets")
|
||||
phases = _array(row["phases"], "runtime phases")
|
||||
blockers = _array(row["blockers"], "runtime blockers")
|
||||
return PortableWorkerRuntimeCandidate(
|
||||
adapter_id=_string(row["adapter_id"], "runtime adapter id"),
|
||||
setup_id=_string(row["setup_id"], "setup id"),
|
||||
definition_id=_string(row["definition_id"], "definition id"),
|
||||
definition_version=_integer(row["definition_version"], "definition version"),
|
||||
definition_sha256=_string(row["definition_sha256"], "definition sha256"),
|
||||
source_adapter_sha256=_string(
|
||||
row["source_adapter_sha256"], "source adapter sha256"
|
||||
),
|
||||
model_manifest_sha256=_string(
|
||||
row["model_manifest_sha256"], "model manifest sha256"
|
||||
),
|
||||
resource_profile_sha256=_string(
|
||||
row["resource_profile_sha256"], "resource profile sha256"
|
||||
),
|
||||
result_contract_sha256=_string(
|
||||
row["result_contract_sha256"], "result contract sha256"
|
||||
),
|
||||
state=state,
|
||||
executor=executor,
|
||||
reusable_assets=tuple(_asset(item) for item in assets),
|
||||
phases=tuple(_phase(item) for item in phases),
|
||||
blockers=tuple(_string(item, "runtime blocker") for item in blockers),
|
||||
candidate_sha256=_string(row["candidate_sha256"], "runtime candidate sha256"),
|
||||
)
|
||||
|
||||
|
||||
def _asset(value: object) -> PortableWorkerAssetRequirement:
|
||||
row = _object(value, "runtime asset")
|
||||
_exact_keys(
|
||||
row,
|
||||
{
|
||||
"asset_id",
|
||||
"kind",
|
||||
"sha256",
|
||||
"byte_length",
|
||||
"component_id",
|
||||
"model_release_id",
|
||||
"model_artifact_role",
|
||||
},
|
||||
"runtime asset",
|
||||
)
|
||||
kind = row["kind"]
|
||||
if kind not in (
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"model-artifact",
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||
return PortableWorkerAssetRequirement(
|
||||
asset_id=_string(row["asset_id"], "runtime asset id"),
|
||||
kind=kind,
|
||||
sha256=_string(row["sha256"], "runtime asset sha256"),
|
||||
byte_length=_optional_integer(row["byte_length"], "runtime asset byte length"),
|
||||
component_id=_optional_string(row["component_id"], "definition component id"),
|
||||
model_release_id=_optional_string(row["model_release_id"], "model release id"),
|
||||
model_artifact_role=_optional_string(
|
||||
row["model_artifact_role"], "model artifact role"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _phase(value: object) -> PortableWorkerRuntimePhase:
|
||||
row = _object(value, "runtime phase")
|
||||
_exact_keys(row, {"phase_id", "state"}, "runtime phase")
|
||||
state = row["state"]
|
||||
if state not in ("implemented", "missing"):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime phase state is invalid")
|
||||
return PortableWorkerRuntimePhase(
|
||||
phase_id=_string(row["phase_id"], "runtime phase id"),
|
||||
state=state,
|
||||
)
|
||||
|
||||
|
||||
def _executor(value: object) -> PortableWorkerExecutorSeal:
|
||||
row = _object(value, "runtime executor")
|
||||
_exact_keys(
|
||||
row,
|
||||
{"release_id", "release_sha256", "image_sha256"},
|
||||
"runtime executor",
|
||||
)
|
||||
return PortableWorkerExecutorSeal(
|
||||
release_id=_string(row["release_id"], "executor release id"),
|
||||
release_sha256=_string(row["release_sha256"], "executor release sha256"),
|
||||
image_sha256=_string(row["image_sha256"], "executor image sha256"),
|
||||
)
|
||||
|
||||
|
||||
def _reject_unsafe_keys(value: object, *, parent: str = "registry") -> None:
|
||||
"""Keep executable instructions and local locators out of shared config."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
if not isinstance(key, str):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime registry object keys must be strings"
|
||||
)
|
||||
normalized = key.lower().replace("-", "_")
|
||||
if (
|
||||
normalized == "path"
|
||||
or normalized.endswith("_path")
|
||||
or normalized in {"command", "commands", "argv", "env", "environment"}
|
||||
or normalized.startswith("command_")
|
||||
or normalized.endswith("_command")
|
||||
or "priority" in normalized
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
f"runtime registry forbids {key!r} in {parent}"
|
||||
)
|
||||
_reject_unsafe_keys(child, parent=key)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
_reject_unsafe_keys(child, parent=parent)
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _array(value: object, label: str) -> list[object]:
|
||||
if not isinstance(value, list):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} must be an array")
|
||||
return value
|
||||
|
||||
|
||||
def _exact_keys(row: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||
if set(row) != expected:
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} fields are invalid")
|
||||
|
||||
|
||||
def _string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_string(value: object, label: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return _string(value, label)
|
||||
|
||||
|
||||
def _integer(value: object, label: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_integer(value: object, label: str) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
return _integer(value, label)
|
||||
|
||||
|
||||
def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None:
|
||||
if pattern.fullmatch(value) is None:
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
|
||||
|
||||
|
||||
def _optional_identifier(value: str | None, label: str) -> None:
|
||||
if value is not None:
|
||||
_pattern(value, _IDENTIFIER, label)
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> None:
|
||||
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
|
||||
@@ -45,6 +45,9 @@ MAX_LIVE_LEASES: Final = 10_000
|
||||
MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024
|
||||
RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1
|
||||
_SQLITE_BUSY_TIMEOUT_MILLISECONDS: Final = 100
|
||||
DEFAULT_RECORDED_CLAIM_LEASE_SECONDS: Final = 120
|
||||
MIN_RECORDED_CLAIM_LEASE_SECONDS: Final = 5
|
||||
MAX_RECORDED_CLAIM_LEASE_SECONDS: Final = 3_600
|
||||
|
||||
LIVE_K1_PRIORITY_RANK: Final = 0
|
||||
RECORDED_PRIORITY_RANK: Final = 100
|
||||
@@ -120,6 +123,11 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
||||
claim_generation INTEGER NOT NULL CHECK (claim_generation >= 0),
|
||||
active_claim_token TEXT,
|
||||
active_claimant_id TEXT,
|
||||
claimed_at_utc TEXT,
|
||||
claim_expires_at_utc TEXT,
|
||||
claim_heartbeat_at_utc TEXT,
|
||||
claim_renewal_count INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (claim_renewal_count >= 0),
|
||||
last_checkpoint_id TEXT,
|
||||
restart_from_zero INTEGER NOT NULL CHECK (restart_from_zero IN (0, 1)),
|
||||
preemption_receipt_sha256 TEXT,
|
||||
@@ -386,6 +394,10 @@ class ObservatoryRecordedJob:
|
||||
claim_generation: int
|
||||
active_claim_token: str | None
|
||||
active_claimant_id: str | None
|
||||
claimed_at_utc: str | None
|
||||
claim_expires_at_utc: str | None
|
||||
claim_heartbeat_at_utc: str | None
|
||||
claim_renewal_count: int
|
||||
last_checkpoint_id: str | None
|
||||
restart_from_zero: bool
|
||||
preemption_receipt_sha256: str | None
|
||||
@@ -461,6 +473,46 @@ class ObservatoryRecordedJob:
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
|
||||
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
|
||||
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
|
||||
lease_values = (
|
||||
self.claimed_at_utc,
|
||||
self.claim_expires_at_utc,
|
||||
self.claim_heartbeat_at_utc,
|
||||
)
|
||||
if (self.active_claim_token is None) != (self.active_claimant_id is None):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim ownership is partial"
|
||||
)
|
||||
if self.active_claim_token is None:
|
||||
if any(value is not None for value in lease_values) or self.claim_renewal_count != 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"inactive recorded-job claim retains lease state"
|
||||
)
|
||||
else:
|
||||
if self.state not in {"claimed", "running", "paused", "preemption-pending"}:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim is active outside an owned state"
|
||||
)
|
||||
if self.claim_generation < 1 or any(value is None for value in lease_values):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"active recorded-job claim has no complete lease"
|
||||
)
|
||||
if self.claim_renewal_count < 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim renewal count is invalid"
|
||||
)
|
||||
assert self.claimed_at_utc is not None
|
||||
assert self.claim_expires_at_utc is not None
|
||||
assert self.claim_heartbeat_at_utc is not None
|
||||
claimed_at = _parse_timestamp(self.claimed_at_utc, "claim timestamp")
|
||||
expires_at = _parse_timestamp(self.claim_expires_at_utc, "claim expiry")
|
||||
heartbeat_at = _parse_timestamp(
|
||||
self.claim_heartbeat_at_utc,
|
||||
"claim heartbeat timestamp",
|
||||
)
|
||||
if not claimed_at <= heartbeat_at < expires_at:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim lease chronology is invalid"
|
||||
)
|
||||
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
|
||||
if not isinstance(self.restart_from_zero, bool):
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
|
||||
@@ -532,6 +584,16 @@ class ObservatoryRecordedJob:
|
||||
"restart_from_zero": self.restart_from_zero,
|
||||
"preemption_receipt_sha256": self.preemption_receipt_sha256,
|
||||
"claim_generation": self.claim_generation,
|
||||
"claim_lease": (
|
||||
None
|
||||
if self.active_claim_token is None
|
||||
else {
|
||||
"claimed_at_utc": self.claimed_at_utc,
|
||||
"expires_at_utc": self.claim_expires_at_utc,
|
||||
"heartbeat_at_utc": self.claim_heartbeat_at_utc,
|
||||
"renewal_count": self.claim_renewal_count,
|
||||
}
|
||||
),
|
||||
"result": (
|
||||
None
|
||||
if self.result_id is None
|
||||
@@ -794,10 +856,12 @@ class ObservatoryRecordedJobQueue:
|
||||
max_jobs: int = MAX_RECORDED_JOBS,
|
||||
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
|
||||
max_live_leases: int = MAX_LIVE_LEASES,
|
||||
claim_lease_seconds: int = DEFAULT_RECORDED_CLAIM_LEASE_SECONDS,
|
||||
) -> None:
|
||||
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
|
||||
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
|
||||
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
|
||||
_validate_claim_lease_seconds(claim_lease_seconds)
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
|
||||
self._definitions = definitions
|
||||
@@ -806,6 +870,7 @@ class ObservatoryRecordedJobQueue:
|
||||
self._max_jobs = max_jobs
|
||||
self._max_claim_receipts = max_claim_receipts
|
||||
self._max_live_leases = max_live_leases
|
||||
self._claim_lease_seconds = claim_lease_seconds
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
@@ -946,6 +1011,8 @@ class ObservatoryRecordedJobQueue:
|
||||
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
|
||||
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
||||
with self._transaction() as connection:
|
||||
now = self._timestamp()
|
||||
self._recover_stale_claims(connection, now=now)
|
||||
receipt = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
|
||||
(claim_request_id,),
|
||||
@@ -976,7 +1043,6 @@ class ObservatoryRecordedJobQueue:
|
||||
"WHERE state = 'queued' "
|
||||
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
|
||||
).fetchone()
|
||||
now = self._timestamp()
|
||||
if row is None:
|
||||
connection.execute(
|
||||
"INSERT INTO observatory_recorded_claim_receipts "
|
||||
@@ -989,12 +1055,26 @@ class ObservatoryRecordedJobQueue:
|
||||
claim_token = hashlib.sha256(
|
||||
f"{uuid4().hex}:{job_id}:{claim_request_id}".encode()
|
||||
).hexdigest()
|
||||
claim_expires_at = _timestamp_after_seconds(
|
||||
now,
|
||||
self._claim_lease_seconds,
|
||||
)
|
||||
updated = connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = 'claimed', "
|
||||
"claim_generation = claim_generation + 1, active_claim_token = ?, "
|
||||
"active_claimant_id = ?, preemption_requested = 0, "
|
||||
"active_claimant_id = ?, claimed_at_utc = ?, "
|
||||
"claim_expires_at_utc = ?, claim_heartbeat_at_utc = ?, "
|
||||
"claim_renewal_count = 0, preemption_requested = 0, "
|
||||
"updated_at_utc = ? WHERE job_id = ? AND state = 'queued'",
|
||||
(claim_token, claimant_id, now, job_id),
|
||||
(
|
||||
claim_token,
|
||||
claimant_id,
|
||||
now,
|
||||
claim_expires_at,
|
||||
now,
|
||||
now,
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
if updated.rowcount != 1:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
@@ -1021,12 +1101,125 @@ class ObservatoryRecordedJobQueue:
|
||||
job=self._get_job(connection, job_id),
|
||||
)
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
heartbeat_sequence: int,
|
||||
) -> ObservatoryRecordedJob:
|
||||
"""Renew one exact active claim using an idempotent heartbeat sequence."""
|
||||
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
_validate_positive_int(claim_generation, "claim generation")
|
||||
_validate_positive_int(heartbeat_sequence, "heartbeat sequence")
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
if job.claim_generation != claim_generation:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim generation is stale"
|
||||
)
|
||||
if job.state not in {"claimed", "running", "paused", "preemption-pending"}:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
f"cannot renew recorded-job claim from {job.state}"
|
||||
)
|
||||
if heartbeat_sequence == job.claim_renewal_count:
|
||||
return job
|
||||
if heartbeat_sequence != job.claim_renewal_count + 1:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded-job heartbeat sequence is not contiguous"
|
||||
)
|
||||
expires_at = _timestamp_after_seconds(now, self._claim_lease_seconds)
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET claim_expires_at_utc = ?, "
|
||||
"claim_heartbeat_at_utc = ?, claim_renewal_count = ?, "
|
||||
"updated_at_utc = ? WHERE job_id = ? AND active_claim_token = ? "
|
||||
"AND claim_generation = ?",
|
||||
(
|
||||
expires_at,
|
||||
now,
|
||||
heartbeat_sequence,
|
||||
now,
|
||||
job_id,
|
||||
claim_token,
|
||||
claim_generation,
|
||||
),
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def authorize_claim_access(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
claimant_id: str,
|
||||
allowed_states: tuple[RecordedJobState, ...] = ("claimed", "running"),
|
||||
) -> ObservatoryRecordedJob:
|
||||
"""Authorize one bounded side-channel operation for the active lease.
|
||||
|
||||
Source downloads and result staging are deliberately not queue state
|
||||
transitions, but they still must be fenced by the exact claimant,
|
||||
token, generation, unexpired lease and an explicitly admitted queue
|
||||
state. Keeping this check inside the queue transaction prevents an
|
||||
artifact transport from reimplementing only part of claim semantics.
|
||||
"""
|
||||
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
_validate_positive_int(claim_generation, "claim generation")
|
||||
_validate_pattern(claimant_id, _IDENTIFIER, "claimant id")
|
||||
if (
|
||||
not allowed_states
|
||||
or len(set(allowed_states)) != len(allowed_states)
|
||||
or any(state not in _recorded_job_states() for state in allowed_states)
|
||||
):
|
||||
raise ValueError("claim-access states are invalid")
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
self._require_active_claim(job, claim_token, now=self._timestamp())
|
||||
if (
|
||||
job.claim_generation != claim_generation
|
||||
or job.active_claimant_id != claimant_id
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim ownership is stale"
|
||||
)
|
||||
if job.state not in allowed_states:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
f"claim-bound access is unavailable from {job.state}"
|
||||
)
|
||||
return job
|
||||
|
||||
def recover_stale_claims(self) -> tuple[ObservatoryRecordedJob, ...]:
|
||||
"""Recover expired ownership without creating a second physical owner.
|
||||
|
||||
A never-started or durably paused claim is safe to requeue. Expired
|
||||
running ownership is quarantined for scheduler reconciliation because
|
||||
lease expiry alone does not prove that its Worker process stopped.
|
||||
"""
|
||||
|
||||
with self._transaction() as connection:
|
||||
recovered_ids = self._recover_stale_claims(
|
||||
connection,
|
||||
now=self._timestamp(),
|
||||
)
|
||||
return tuple(self._get_job(connection, job_id) for job_id in recovered_ids)
|
||||
|
||||
def start(self, job_id: str, *, claim_token: str) -> ObservatoryRecordedJob:
|
||||
"""Enter running state, or yield before execution when live has priority."""
|
||||
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
self._require_active_claim(job, claim_token)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
if job.state == "running":
|
||||
return job
|
||||
if job.state == "paused":
|
||||
@@ -1039,7 +1232,7 @@ class ObservatoryRecordedJobQueue:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
(state, int(state == "paused"), self._timestamp(), job_id),
|
||||
(state, int(state == "paused"), now, job_id),
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
@@ -1053,9 +1246,11 @@ class ObservatoryRecordedJobQueue:
|
||||
"""Record an allowlisted cooperative boundary and yield if live is open."""
|
||||
|
||||
_validate_pattern(checkpoint_id, _CHECKPOINT_ID, "checkpoint id")
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
self._require_active_claim(job, claim_token)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
if job.checkpoint_policy != "cooperative":
|
||||
raise ObservatoryRecordedCheckpointError(
|
||||
"recorded RunDefinition is non-checkpointable"
|
||||
@@ -1079,7 +1274,7 @@ class ObservatoryRecordedJobQueue:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||
"last_checkpoint_id = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
(state, checkpoint_id, self._timestamp(), job_id),
|
||||
(state, checkpoint_id, now, job_id),
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
@@ -1195,6 +1390,7 @@ class ObservatoryRecordedJobQueue:
|
||||
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
||||
"""Close recorded admission without allowing a monolith to delay live K1."""
|
||||
|
||||
self.recover_stale_claims()
|
||||
created = False
|
||||
with self._transaction() as connection:
|
||||
existing = connection.execute(
|
||||
@@ -1341,6 +1537,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"""Activate only after every recorded resource owner has yielded."""
|
||||
|
||||
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
lease = self._get_live_lease(connection, lease_id)
|
||||
if lease.state == "active":
|
||||
@@ -1438,7 +1635,9 @@ class ObservatoryRecordedJobQueue:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = 'queued', "
|
||||
"preemption_requested = 0, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, updated_at_utc = ? "
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, updated_at_utc = ? "
|
||||
"WHERE state = 'paused'",
|
||||
(now,),
|
||||
)
|
||||
@@ -1488,22 +1687,34 @@ class ObservatoryRecordedJobQueue:
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
token_sha256 = hashlib.sha256(claim_token.encode()).hexdigest()
|
||||
self.recover_stale_claims()
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
if job.state in _TERMINAL_STATES:
|
||||
exact_replay = (
|
||||
job.state == state
|
||||
and job.result_id == result_id
|
||||
and job.result_sha256 == result_sha256
|
||||
and job.terminal_code == terminal_code
|
||||
and job.terminal_message == terminal_message
|
||||
and job.terminal_claim_token_sha256 == token_sha256
|
||||
)
|
||||
if exact_replay:
|
||||
return job
|
||||
if (
|
||||
job.state != state
|
||||
or job.result_id != result_id
|
||||
or job.result_sha256 != result_sha256
|
||||
or job.terminal_code != terminal_code
|
||||
or job.terminal_message != terminal_message
|
||||
or job.terminal_claim_token_sha256 != token_sha256
|
||||
job.terminal_claim_token_sha256 != token_sha256
|
||||
or job.terminal_code
|
||||
in {"claim-lease-expired", "claim-lease-migration"}
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job terminal acknowledgement is stale"
|
||||
)
|
||||
else:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded job is bound to another terminal outcome"
|
||||
)
|
||||
return job
|
||||
self._require_active_claim(job, claim_token)
|
||||
now = self._timestamp()
|
||||
self._require_active_claim(job, claim_token, now=now)
|
||||
allowed_states = (
|
||||
("running",)
|
||||
if state == "succeeded"
|
||||
@@ -1523,7 +1734,9 @@ class ObservatoryRecordedJobQueue:
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, result_id = ?, "
|
||||
"result_sha256 = ?, terminal_code = ?, terminal_message = ?, "
|
||||
"terminal_claim_token_sha256 = ?, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, updated_at_utc = ? WHERE job_id = ?",
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?",
|
||||
(
|
||||
state,
|
||||
result_id,
|
||||
@@ -1531,7 +1744,7 @@ class ObservatoryRecordedJobQueue:
|
||||
terminal_code,
|
||||
terminal_message,
|
||||
token_sha256,
|
||||
self._timestamp(),
|
||||
now,
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
@@ -1548,18 +1761,105 @@ class ObservatoryRecordedJobQueue:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored claim receipt has a partial job identity"
|
||||
)
|
||||
job = self._get_job(connection, job_id)
|
||||
if job.active_claim_token != claim_token:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim receipt no longer owns the job"
|
||||
)
|
||||
return ObservatoryRecordedClaim(
|
||||
claim_request_id=str(receipt["claim_request_id"]),
|
||||
request_sha256=str(receipt["request_sha256"]),
|
||||
claimant_id=str(receipt["claimant_id"]),
|
||||
claim_token=claim_token,
|
||||
job=self._get_job(connection, job_id),
|
||||
job=job,
|
||||
)
|
||||
|
||||
def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None:
|
||||
def _require_active_claim(
|
||||
self,
|
||||
job: ObservatoryRecordedJob,
|
||||
claim_token: str,
|
||||
*,
|
||||
now: str | None = None,
|
||||
) -> None:
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
if job.active_claim_token != claim_token:
|
||||
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
|
||||
if now is not None and (
|
||||
job.claim_expires_at_utc is None
|
||||
or _parse_timestamp(now, "queue timestamp")
|
||||
>= _parse_timestamp(job.claim_expires_at_utc, "claim expiry")
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim lease expired"
|
||||
)
|
||||
|
||||
def _recover_stale_claims(
|
||||
self,
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
now: str,
|
||||
) -> tuple[str, ...]:
|
||||
now_value = _parse_timestamp(now, "queue timestamp")
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs "
|
||||
"WHERE active_claim_token IS NOT NULL "
|
||||
"ORDER BY created_at_utc, job_id"
|
||||
).fetchall()
|
||||
recovered: list[str] = []
|
||||
live_open = self._open_live_lease_row(connection) is not None
|
||||
for row in rows:
|
||||
job = _job_from_row(row)
|
||||
if job.claim_expires_at_utc is None:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"active recorded-job claim has no expiry"
|
||||
)
|
||||
if _parse_timestamp(job.claim_expires_at_utc, "claim expiry") > now_value:
|
||||
continue
|
||||
assert job.active_claim_token is not None
|
||||
token_sha256 = hashlib.sha256(job.active_claim_token.encode()).hexdigest()
|
||||
if job.state in {"claimed", "paused"}:
|
||||
next_state = "paused" if job.state == "paused" and live_open else "queued"
|
||||
restart_from_zero = job.restart_from_zero or job.state == "paused"
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||
"preemption_requested = ?, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, last_checkpoint_id = ?, "
|
||||
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
(
|
||||
next_state,
|
||||
int(next_state == "paused"),
|
||||
None if restart_from_zero else job.last_checkpoint_id,
|
||||
int(restart_from_zero),
|
||||
now,
|
||||
job.job_id,
|
||||
),
|
||||
)
|
||||
elif job.state in {"running", "preemption-pending"}:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs "
|
||||
"SET state = 'reconciliation-required', result_id = NULL, "
|
||||
"result_sha256 = NULL, terminal_code = 'claim-lease-expired', "
|
||||
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||
"updated_at_utc = ? WHERE job_id = ?",
|
||||
(
|
||||
"Worker claim lease expired after execution started; "
|
||||
"physical resource ownership requires reconciliation.",
|
||||
token_sha256,
|
||||
now,
|
||||
job.job_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"active recorded-job claim is stored in an invalid state"
|
||||
)
|
||||
recovered.append(job.job_id)
|
||||
return tuple(recovered)
|
||||
|
||||
def _cancellation_request(
|
||||
self,
|
||||
@@ -1626,7 +1926,9 @@ class ObservatoryRecordedJobQueue:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = 'paused', "
|
||||
"preemption_requested = 1, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, last_checkpoint_id = NULL, "
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
|
||||
"restart_from_zero = 1, preemption_receipt_sha256 = ?, "
|
||||
"updated_at_utc = ? WHERE job_id = ?",
|
||||
(receipt.receipt_sha256, now, job.job_id),
|
||||
@@ -1638,6 +1940,8 @@ class ObservatoryRecordedJobQueue:
|
||||
"result_sha256 = NULL, terminal_code = 'preemption-race', "
|
||||
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||
"preemption_receipt_sha256 = ?, updated_at_utc = ? "
|
||||
"WHERE job_id = ?",
|
||||
(
|
||||
@@ -1707,6 +2011,7 @@ class ObservatoryRecordedJobQueue:
|
||||
self.data_dir.chmod(0o700)
|
||||
with self._connect() as connection:
|
||||
connection.executescript(_SCHEMA_SQL)
|
||||
self._migrate_claim_lease_schema(connection)
|
||||
self._validate_schema(connection)
|
||||
self._validate_existing_capacity(connection)
|
||||
connection.commit()
|
||||
@@ -1723,7 +2028,7 @@ class ObservatoryRecordedJobQueue:
|
||||
|
||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||
expected = {
|
||||
"observatory_recorded_jobs": 42,
|
||||
"observatory_recorded_jobs": 46,
|
||||
"observatory_recorded_claim_receipts": 6,
|
||||
"observatory_live_leases": 13,
|
||||
"observatory_recorded_preemptions": 14,
|
||||
@@ -1734,6 +2039,21 @@ class ObservatoryRecordedJobQueue:
|
||||
).fetchall()
|
||||
if len(columns) != column_count:
|
||||
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
|
||||
job_columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
|
||||
).fetchall()
|
||||
}
|
||||
if not {
|
||||
"claimed_at_utc",
|
||||
"claim_expires_at_utc",
|
||||
"claim_heartbeat_at_utc",
|
||||
"claim_renewal_count",
|
||||
}.issubset(job_columns):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim lease schema is unavailable"
|
||||
)
|
||||
indexes = connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'index' "
|
||||
"AND name = 'observatory_one_open_live_lease'"
|
||||
@@ -1743,6 +2063,75 @@ class ObservatoryRecordedJobQueue:
|
||||
"live K1 lease exclusivity index is unavailable"
|
||||
)
|
||||
|
||||
def _migrate_claim_lease_schema(self, connection: sqlite3.Connection) -> None:
|
||||
"""Add renewable claim columns and safely fence legacy active owners."""
|
||||
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
|
||||
).fetchall()
|
||||
}
|
||||
additions = (
|
||||
("claimed_at_utc", "TEXT"),
|
||||
("claim_expires_at_utc", "TEXT"),
|
||||
("claim_heartbeat_at_utc", "TEXT"),
|
||||
(
|
||||
"claim_renewal_count",
|
||||
"INTEGER NOT NULL DEFAULT 0 CHECK (claim_renewal_count >= 0)",
|
||||
),
|
||||
)
|
||||
missing = [name for name, _definition in additions if name not in columns]
|
||||
if not missing:
|
||||
return
|
||||
for name, definition in additions:
|
||||
if name not in columns:
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}"
|
||||
)
|
||||
|
||||
now = self._timestamp()
|
||||
rows = connection.execute(
|
||||
"SELECT job_id, state, active_claim_token "
|
||||
"FROM observatory_recorded_jobs WHERE active_claim_token IS NOT NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
job_id = str(row["job_id"])
|
||||
state = str(row["state"])
|
||||
claim_token = str(row["active_claim_token"])
|
||||
if state in {"claimed", "paused"}:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = 'queued', "
|
||||
"preemption_requested = 0, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, last_checkpoint_id = NULL, "
|
||||
"restart_from_zero = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
(int(state == "paused"), now, job_id),
|
||||
)
|
||||
elif state in {"running", "preemption-pending"}:
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs "
|
||||
"SET state = 'reconciliation-required', result_id = NULL, "
|
||||
"result_sha256 = NULL, terminal_code = 'claim-lease-migration', "
|
||||
"terminal_message = ?, terminal_claim_token_sha256 = ?, "
|
||||
"active_claim_token = NULL, active_claimant_id = NULL, "
|
||||
"claimed_at_utc = NULL, claim_expires_at_utc = NULL, "
|
||||
"claim_heartbeat_at_utc = NULL, claim_renewal_count = 0, "
|
||||
"updated_at_utc = ? WHERE job_id = ?",
|
||||
(
|
||||
"Legacy Worker execution had no expiring lease; physical "
|
||||
"resource ownership requires reconciliation.",
|
||||
hashlib.sha256(claim_token.encode()).hexdigest(),
|
||||
now,
|
||||
job_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"legacy active claim is stored in an invalid state"
|
||||
)
|
||||
|
||||
def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None:
|
||||
for table, limit, label in (
|
||||
("observatory_recorded_jobs", self._max_jobs, "recorded job"),
|
||||
@@ -1901,6 +2290,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
claim_generation=row["claim_generation"],
|
||||
active_claim_token=row["active_claim_token"],
|
||||
active_claimant_id=row["active_claimant_id"],
|
||||
claimed_at_utc=row["claimed_at_utc"],
|
||||
claim_expires_at_utc=row["claim_expires_at_utc"],
|
||||
claim_heartbeat_at_utc=row["claim_heartbeat_at_utc"],
|
||||
claim_renewal_count=row["claim_renewal_count"],
|
||||
last_checkpoint_id=row["last_checkpoint_id"],
|
||||
restart_from_zero=bool(row["restart_from_zero"]),
|
||||
preemption_receipt_sha256=row["preemption_receipt_sha256"],
|
||||
@@ -2029,6 +2422,17 @@ def _validate_quota(value: object, maximum: int, label: str) -> None:
|
||||
raise ValueError(f"{label} quota is invalid")
|
||||
|
||||
|
||||
def _validate_claim_lease_seconds(value: object) -> None:
|
||||
if (
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or not MIN_RECORDED_CLAIM_LEASE_SECONDS
|
||||
<= value
|
||||
<= MAX_RECORDED_CLAIM_LEASE_SECONDS
|
||||
):
|
||||
raise ValueError("recorded-job claim lease duration is invalid")
|
||||
|
||||
|
||||
def _validate_positive_int(value: object, label: str) -> None:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise ValueError(f"{label} must be positive")
|
||||
@@ -2059,6 +2463,10 @@ def _validate_text(value: object, label: str, *, max_length: int) -> None:
|
||||
|
||||
|
||||
def _validate_timestamp(value: object, label: str) -> None:
|
||||
_parse_timestamp(value, label)
|
||||
|
||||
|
||||
def _parse_timestamp(value: object, label: str) -> datetime:
|
||||
_validate_text(value, label, max_length=64)
|
||||
assert isinstance(value, str)
|
||||
try:
|
||||
@@ -2067,6 +2475,12 @@ def _validate_timestamp(value: object, label: str) -> None:
|
||||
raise ValueError(f"{label} is invalid") from exc
|
||||
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
|
||||
raise ValueError(f"{label} must use UTC")
|
||||
return parsed
|
||||
|
||||
|
||||
def _timestamp_after_seconds(value: str, seconds: int) -> str:
|
||||
expires_at = _parse_timestamp(value, "queue timestamp") + timedelta(seconds=seconds)
|
||||
return expires_at.isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
|
||||
@@ -12,8 +12,11 @@ import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final, Literal
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from k1link.observatory.canonical_result import (
|
||||
is_admitted_observatory_recorded_result,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
|
||||
@@ -22,6 +25,9 @@ LABORATORY_SETUP_REGISTRY_SCHEMA: Final = (
|
||||
LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||
"missioncore.observatory-laboratory-setup-catalog/v1"
|
||||
)
|
||||
OBSERVATORY_CALCULATION_PROFILE_SCHEMA: Final = (
|
||||
"missioncore.observatory-calculation-profile/v1"
|
||||
)
|
||||
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
||||
_MAX_CONFIGURATION_BYTES: Final = 4 * 1024 * 1024
|
||||
_IDENTIFIER: Final = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
@@ -282,6 +288,42 @@ class LaboratorySetupRegistry:
|
||||
return result.result_kind
|
||||
return None
|
||||
|
||||
def observatory_calculation_profile(
|
||||
self,
|
||||
summary: SessionSummary,
|
||||
) -> dict[str, object] | None:
|
||||
"""Project an exact preserved legacy profile without inferring identity."""
|
||||
|
||||
for setup in self.setups:
|
||||
for result in setup.preserved_results:
|
||||
if (
|
||||
result.access != "observatory"
|
||||
or result.result_id != summary.session_id
|
||||
):
|
||||
continue
|
||||
if (
|
||||
setup.origin != "existing-result"
|
||||
or setup.run_definition is not None
|
||||
):
|
||||
return None
|
||||
if not is_admitted_observatory_recorded_result(
|
||||
summary,
|
||||
expected_result_id=result.result_id,
|
||||
expected_source_session_id=setup.source_session_id,
|
||||
expected_result_kind=result.result_kind,
|
||||
):
|
||||
return None
|
||||
return {
|
||||
"schema_version": OBSERVATORY_CALCULATION_PROFILE_SCHEMA,
|
||||
"setup_id": setup.setup_id,
|
||||
"display_name": setup.display_name,
|
||||
"origin": setup.origin,
|
||||
"definition_id": None,
|
||||
"definition_version": None,
|
||||
"definition_sha256": None,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
|
||||
row = _object(value, "setup")
|
||||
@@ -330,7 +372,7 @@ def _setup(value: object, *, repository_root: Path) -> LaboratorySetup:
|
||||
setup_id=_identifier(row["setup_id"], "setup_id"),
|
||||
display_name=_text(row["display_name"], "display_name"),
|
||||
description=_text(row["description"], "description"),
|
||||
origin=origin,
|
||||
origin=cast(SetupOrigin, origin),
|
||||
source_session_id=_text(source["session_id"], "source session_id"),
|
||||
source_label=_text(source["label"], "source label"),
|
||||
required_modalities=modalities,
|
||||
@@ -427,7 +469,7 @@ def _preserved_result(value: object) -> _PreservedResult:
|
||||
result_id=result_id,
|
||||
result_kind=_identifier(row["result_kind"], "result_kind"),
|
||||
relation=_identifier(row["relation"], "result relation"),
|
||||
access=access,
|
||||
access=cast(ResultAccess, access),
|
||||
created_at_utc=_text(row["created_at_utc"], "created_at_utc"),
|
||||
)
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import re
|
||||
import secrets
|
||||
import stat
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
@@ -29,6 +29,7 @@ from k1link.sessions.media import (
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
RecordedMediaArtifact,
|
||||
ReplayArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
@@ -40,6 +41,8 @@ PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bun
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
|
||||
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID: Final = "raw-transport-index"
|
||||
PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE: Final = "application/x-ndjson"
|
||||
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
@@ -384,10 +387,14 @@ class RecordedK1SourceAdmissionService:
|
||||
camera_source=camera_source,
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
sealed_replay = _seal_replay_artifact_digests(
|
||||
detail=detail,
|
||||
replay=replay,
|
||||
)
|
||||
self._verify_replay(
|
||||
detail=detail,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
replay=sealed_replay,
|
||||
)
|
||||
try:
|
||||
media = (
|
||||
@@ -410,7 +417,7 @@ class RecordedK1SourceAdmissionService:
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha256,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
replay=sealed_replay,
|
||||
media=media,
|
||||
)
|
||||
source_bundle = _canonical_json(source_bundle_document)
|
||||
@@ -592,13 +599,31 @@ class RecordedK1SourceAdmissionService:
|
||||
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
|
||||
for replay_artifact in replay.artifacts:
|
||||
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
||||
exact_metadata_member = (
|
||||
catalog_artifact is not None
|
||||
and replay_artifact.artifact_id
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and catalog_artifact.kind
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and replay_artifact.media_type
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
and catalog_artifact.media_type
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
)
|
||||
digest_matches_catalog = (
|
||||
replay_artifact.expected_sha256 == catalog_artifact.sha256
|
||||
if catalog_artifact is not None and catalog_artifact.sha256 is not None
|
||||
else exact_metadata_member
|
||||
and isinstance(replay_artifact.expected_sha256, str)
|
||||
and _SHA256.fullmatch(replay_artifact.expected_sha256) is not None
|
||||
)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||
or replay_artifact.media_type != catalog_artifact.media_type
|
||||
or replay_artifact.file_byte_length != catalog_artifact.byte_length
|
||||
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
|
||||
or replay_artifact.expected_sha256 != catalog_artifact.sha256
|
||||
or not digest_matches_catalog
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay member disagrees with the catalog"
|
||||
@@ -799,6 +824,130 @@ class RecordedK1SourceAdmissionService:
|
||||
}
|
||||
|
||||
|
||||
def _seal_replay_artifact_digests(
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
replay: ReplayCommand,
|
||||
) -> ReplayCommand:
|
||||
"""Seal the one legacy replay member whose catalog has no stored digest.
|
||||
|
||||
Historical K1 catalogs explicitly register ``mqtt.metadata.jsonl`` as the
|
||||
``raw-transport-index`` replay artifact, but the catalog row predates a
|
||||
persisted SHA-256 column value. Portable admission may derive that one
|
||||
digest from the already confined ReplayCommand handle. No sibling-name or
|
||||
directory discovery is permitted, and every other missing digest remains
|
||||
an integrity failure in ``_verify_replay``.
|
||||
"""
|
||||
|
||||
catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
sealed: list[ReplayArtifact] = []
|
||||
sealed_metadata_count = 0
|
||||
for artifact in replay.artifacts:
|
||||
if artifact.expected_sha256 is not None:
|
||||
sealed.append(artifact)
|
||||
continue
|
||||
catalog_artifact = catalog_artifacts.get(artifact.artifact_id)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or artifact.artifact_id
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or catalog_artifact.kind
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
or catalog_artifact.media_type
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
or catalog_artifact.sha256 is not None
|
||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||
or artifact.file_byte_length != catalog_artifact.byte_length
|
||||
or artifact.replay_byte_length != catalog_artifact.byte_length
|
||||
):
|
||||
sealed.append(artifact)
|
||||
continue
|
||||
sealed_metadata_count += 1
|
||||
if sealed_metadata_count != 1:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata member is not unique"
|
||||
)
|
||||
sha256 = _hash_confined_replay_artifact(replay=replay, artifact=artifact)
|
||||
sealed.append(replace(artifact, expected_sha256=sha256))
|
||||
return replace(replay, artifacts=tuple(sealed))
|
||||
|
||||
|
||||
def _hash_confined_replay_artifact(
|
||||
*,
|
||||
replay: ReplayCommand,
|
||||
artifact: ReplayArtifact,
|
||||
) -> str:
|
||||
descriptor = -1
|
||||
try:
|
||||
allowed_root = replay.allowed_root.resolve(strict=True)
|
||||
session_root = replay.session_root.resolve(strict=True)
|
||||
source_metadata = artifact.path.lstat()
|
||||
source_path = artifact.path.resolve(strict=True)
|
||||
if (
|
||||
not allowed_root.is_dir()
|
||||
or not session_root.is_dir()
|
||||
or not session_root.is_relative_to(allowed_root)
|
||||
or not source_path.is_relative_to(session_root)
|
||||
or stat.S_ISLNK(source_metadata.st_mode)
|
||||
or not stat.S_ISREG(source_metadata.st_mode)
|
||||
or not 1 <= artifact.file_byte_length <= MAX_SAFE_INTEGER
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata escapes its admitted session root"
|
||||
)
|
||||
descriptor = os.open(
|
||||
source_path,
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_size != artifact.file_byte_length
|
||||
or before.st_size != artifact.replay_byte_length
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata is outside admitted bounds"
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
byte_length = 0
|
||||
while chunk := os.read(descriptor, 1024 * 1024):
|
||||
byte_length += len(chunk)
|
||||
if byte_length > artifact.file_byte_length:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata grew while it was sealed"
|
||||
)
|
||||
digest.update(chunk)
|
||||
after = os.fstat(descriptor)
|
||||
stable_identity = (
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_size,
|
||||
before.st_mtime_ns,
|
||||
before.st_ctime_ns,
|
||||
) == (
|
||||
after.st_dev,
|
||||
after.st_ino,
|
||||
after.st_size,
|
||||
after.st_mtime_ns,
|
||||
after.st_ctime_ns,
|
||||
)
|
||||
if byte_length != artifact.file_byte_length or not stable_identity:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata changed while it was sealed"
|
||||
)
|
||||
return digest.hexdigest()
|
||||
except PortableSourceAdmissionIntegrityError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay metadata is unavailable"
|
||||
) from exc
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _read_canonical_camera_summary(
|
||||
source_path: Path,
|
||||
) -> tuple[dict[str, object], int]:
|
||||
|
||||
@@ -19,6 +19,7 @@ import re
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -51,6 +52,7 @@ type WorkerCycleState = Literal[
|
||||
"succeeded",
|
||||
"failed",
|
||||
"rejected",
|
||||
"lease-lost",
|
||||
]
|
||||
type RecordedJobWireState = Literal[
|
||||
"accepted",
|
||||
@@ -122,6 +124,7 @@ class SealedObservatoryRecordedJob:
|
||||
job_id: str
|
||||
request_sha256: str
|
||||
identity_sha256: str
|
||||
submission_receipt_sha256: str
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
@@ -140,6 +143,10 @@ class SealedObservatoryRecordedJob:
|
||||
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
|
||||
allowed_checkpoints: tuple[str, ...]
|
||||
claim_generation: int
|
||||
claim_claimed_at_utc: str | None
|
||||
claim_expires_at_utc: str | None
|
||||
claim_heartbeat_at_utc: str | None
|
||||
claim_renewal_count: int
|
||||
restart_from_zero: bool
|
||||
|
||||
|
||||
@@ -213,6 +220,16 @@ class ObservatoryWorkerTransport(Protocol):
|
||||
claim_token: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
claim_generation: int,
|
||||
heartbeat_sequence: int,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
@@ -308,6 +325,13 @@ class _TerminalPayload(_StrictPayload):
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
class _ClaimLeasePayload(_StrictPayload):
|
||||
claimed_at_utc: Timestamp
|
||||
expires_at_utc: Timestamp
|
||||
heartbeat_at_utc: Timestamp
|
||||
renewal_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class _RecordedJobPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
||||
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
||||
@@ -329,6 +353,7 @@ class _RecordedJobPayload(_StrictPayload):
|
||||
restart_from_zero: bool
|
||||
preemption_receipt_sha256: Sha256 | None
|
||||
claim_generation: int = Field(ge=0)
|
||||
claim_lease: _ClaimLeasePayload | None
|
||||
result: _ResultPayload | None
|
||||
terminal: _TerminalPayload | None
|
||||
created_at_utc: Timestamp
|
||||
@@ -365,10 +390,20 @@ class ObservatoryWorkerAgent:
|
||||
transport: ObservatoryWorkerTransport,
|
||||
executors: ObservatoryWorkerExecutorRegistry,
|
||||
claim_request_id_factory: Callable[[], str] | None = None,
|
||||
heartbeat_interval_seconds: float | None = None,
|
||||
heartbeat_stop_timeout_seconds: float = 5.0,
|
||||
) -> None:
|
||||
if heartbeat_interval_seconds is not None and not (
|
||||
0.01 <= heartbeat_interval_seconds <= 300.0
|
||||
):
|
||||
raise ValueError("Worker heartbeat interval is invalid")
|
||||
if not 0.1 <= heartbeat_stop_timeout_seconds <= 300.0:
|
||||
raise ValueError("Worker heartbeat stop timeout is invalid")
|
||||
self._transport = transport
|
||||
self._executors = executors
|
||||
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
|
||||
self._heartbeat_interval_seconds = heartbeat_interval_seconds
|
||||
self._heartbeat_stop_timeout_seconds = heartbeat_stop_timeout_seconds
|
||||
self._cycle_lock = threading.Lock()
|
||||
|
||||
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||
@@ -443,11 +478,33 @@ class ObservatoryWorkerAgent:
|
||||
job_id=claim.job.job_id,
|
||||
)
|
||||
|
||||
active_job = _seal_job(started)
|
||||
heartbeat = _ClaimHeartbeat(
|
||||
transport=self._transport,
|
||||
job=active_job,
|
||||
claim_token=claim.claim_token,
|
||||
interval_seconds=(
|
||||
self._heartbeat_interval_seconds
|
||||
if self._heartbeat_interval_seconds is not None
|
||||
else _default_heartbeat_interval(active_job)
|
||||
),
|
||||
stop_timeout_seconds=self._heartbeat_stop_timeout_seconds,
|
||||
)
|
||||
heartbeat.start()
|
||||
|
||||
try:
|
||||
result = adapter.execute(claim.job)
|
||||
result = adapter.execute(active_job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
heartbeat.stop()
|
||||
if heartbeat.failed:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="lease-lost",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code="claim-heartbeat-lost",
|
||||
)
|
||||
failure_code = "executor-error"
|
||||
acknowledgement = self._transport.fail(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
@@ -468,6 +525,15 @@ class ObservatoryWorkerAgent:
|
||||
failure_code=failure_code,
|
||||
)
|
||||
|
||||
heartbeat.stop()
|
||||
if heartbeat.failed:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="lease-lost",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code="claim-heartbeat-lost",
|
||||
)
|
||||
|
||||
acknowledgement = self._transport.succeed(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
@@ -495,6 +561,92 @@ class ObservatoryWorkerAgent:
|
||||
)
|
||||
|
||||
|
||||
class _ClaimHeartbeat:
|
||||
"""Renew one exact generation while an executor owns Worker resources."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
transport: ObservatoryWorkerTransport,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
claim_token: str,
|
||||
interval_seconds: float,
|
||||
stop_timeout_seconds: float,
|
||||
) -> None:
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("Worker heartbeat interval must be positive")
|
||||
self._transport = transport
|
||||
self._job = job
|
||||
self._claim_token = claim_token
|
||||
self._interval_seconds = interval_seconds
|
||||
self._stop_timeout_seconds = stop_timeout_seconds
|
||||
self._stop = threading.Event()
|
||||
self._state_lock = threading.Lock()
|
||||
self._failure: Exception | None = None
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"observatory-heartbeat-{job.job_id}",
|
||||
daemon=True,
|
||||
)
|
||||
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
with self._state_lock:
|
||||
return self._failure is not None
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=self._stop_timeout_seconds)
|
||||
if self._thread.is_alive():
|
||||
self._record_failure(
|
||||
ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim heartbeat did not stop within its bound"
|
||||
)
|
||||
)
|
||||
|
||||
def _run(self) -> None:
|
||||
sequence = self._job.claim_renewal_count + 1
|
||||
# Renew immediately once start has been acknowledged. Waiting a full
|
||||
# interval here would assume that claim/start transport latency consumed
|
||||
# none of the original lease window.
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
acknowledgement = self._transport.renew_claim(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=self._job.job_id,
|
||||
claim_token=self._claim_token,
|
||||
claim_generation=self._job.claim_generation,
|
||||
heartbeat_sequence=sequence,
|
||||
)
|
||||
renewed = _validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=self._job,
|
||||
expected_state=("claimed", "running"),
|
||||
)
|
||||
if (
|
||||
renewed.claim_lease is None
|
||||
or renewed.claim_lease.renewal_count != sequence
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker heartbeat acknowledgement changed its sequence"
|
||||
)
|
||||
sequence += 1
|
||||
except Exception as exc:
|
||||
self._record_failure(exc)
|
||||
self._stop.set()
|
||||
return
|
||||
if self._stop.wait(self._interval_seconds):
|
||||
return
|
||||
|
||||
def _record_failure(self, exc: Exception) -> None:
|
||||
with self._state_lock:
|
||||
if self._failure is None:
|
||||
self._failure = exc
|
||||
|
||||
|
||||
def _validate_claim(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
@@ -519,6 +671,10 @@ def _validate_claim(
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim job is not in a claimed generation"
|
||||
)
|
||||
if claim.job.claim_lease is None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim has no renewable lease"
|
||||
)
|
||||
if claim.job.result is not None or claim.job.terminal is not None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim already carries a terminal outcome"
|
||||
@@ -556,6 +712,14 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
||||
and payload.checkpoint_policy.allowed_checkpoints
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
|
||||
if payload.claim_lease is not None:
|
||||
claimed_at = _parse_timestamp(payload.claim_lease.claimed_at_utc)
|
||||
expires_at = _parse_timestamp(payload.claim_lease.expires_at_utc)
|
||||
heartbeat_at = _parse_timestamp(payload.claim_lease.heartbeat_at_utc)
|
||||
if not claimed_at <= heartbeat_at < expires_at:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim lease chronology is inconsistent"
|
||||
)
|
||||
|
||||
expected_request_sha256 = _sha256_document(
|
||||
{
|
||||
@@ -615,6 +779,7 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
||||
job_id=payload.job_id,
|
||||
request_sha256=payload.request_sha256,
|
||||
identity_sha256=payload.identity_sha256,
|
||||
submission_receipt_sha256=payload.submission_receipt_sha256,
|
||||
source_session_id=payload.source.session_id,
|
||||
source_catalog_sha256=payload.source.catalog_sha256,
|
||||
source_bundle_sha256=payload.source.bundle_sha256,
|
||||
@@ -638,6 +803,18 @@ def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
||||
checkpoint_policy=payload.checkpoint_policy.mode,
|
||||
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
|
||||
claim_generation=payload.claim_generation,
|
||||
claim_claimed_at_utc=(
|
||||
None if payload.claim_lease is None else payload.claim_lease.claimed_at_utc
|
||||
),
|
||||
claim_expires_at_utc=(
|
||||
None if payload.claim_lease is None else payload.claim_lease.expires_at_utc
|
||||
),
|
||||
claim_heartbeat_at_utc=(
|
||||
None if payload.claim_lease is None else payload.claim_lease.heartbeat_at_utc
|
||||
),
|
||||
claim_renewal_count=(
|
||||
0 if payload.claim_lease is None else payload.claim_lease.renewal_count
|
||||
),
|
||||
restart_from_zero=payload.restart_from_zero,
|
||||
)
|
||||
|
||||
@@ -662,6 +839,12 @@ def _validate_transition_acknowledgement(
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement has an unexpected state"
|
||||
)
|
||||
if acknowledgement.state in {"claimed", "running", "preemption-pending"} and (
|
||||
acknowledgement.claim_lease is None
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement lost the active claim lease"
|
||||
)
|
||||
if (
|
||||
sealed.job_id != expected_job.job_id
|
||||
or sealed.identity_sha256 != expected_job.identity_sha256
|
||||
@@ -677,6 +860,20 @@ def _default_claim_request_id() -> str:
|
||||
return f"worker-006:{uuid4().hex}"
|
||||
|
||||
|
||||
def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float:
|
||||
if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim has no heartbeat lease bounds"
|
||||
)
|
||||
remaining = (
|
||||
_parse_timestamp(job.claim_expires_at_utc)
|
||||
- _parse_timestamp(job.claim_heartbeat_at_utc)
|
||||
).total_seconds()
|
||||
if remaining <= 0:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired")
|
||||
return max(0.5, min(30.0, remaining / 3.0))
|
||||
|
||||
|
||||
def _bounded_executor_failure(exc: Exception) -> str:
|
||||
detail = " ".join(str(exc).split())
|
||||
message = f"Executor adapter raised {type(exc).__name__}."
|
||||
@@ -693,3 +890,7 @@ def _sha256_document(document: Mapping[str, object]) -> str:
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,357 @@
|
||||
"""Install-time composition and bounded polling for portable Worker 006.
|
||||
|
||||
The durable service owns transport cadence only. A reviewed Worker release
|
||||
must inject an in-memory executor registry whose four-digest identities cover
|
||||
every portable RunDefinition advertised as ready. Configuration cannot name
|
||||
Python modules, commands, images, or executable paths, so neither Mission Core
|
||||
nor an environment variable can turn an unsealed candidate into code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerAgent,
|
||||
ObservatoryWorkerCycleReport,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerExecutorRegistry,
|
||||
ObservatoryWorkerExecutorUnavailableError,
|
||||
)
|
||||
from k1link.observatory.worker_http_transport import (
|
||||
ObservatoryWorkerHttpError,
|
||||
ObservatoryWorkerHttpGateway,
|
||||
)
|
||||
|
||||
OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"
|
||||
OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"
|
||||
OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"
|
||||
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS"
|
||||
)
|
||||
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS"
|
||||
)
|
||||
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES"
|
||||
)
|
||||
|
||||
DEFAULT_OBSERVATORY_WORKER_BASE_URL: Final = "http://127.0.0.1:18080"
|
||||
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS: Final = 1.0
|
||||
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS: Final = 5.0
|
||||
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES: Final = 12
|
||||
|
||||
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
|
||||
|
||||
|
||||
class ObservatoryWorkerServiceError(RuntimeError):
|
||||
"""Worker 006 cannot start without its exact local operational boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerServiceConfiguration:
|
||||
"""Path-only service configuration; it never contains a plaintext secret."""
|
||||
|
||||
base_url: str
|
||||
bearer_token_file: Path
|
||||
work_root: Path
|
||||
idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS
|
||||
transport_backoff_seconds: float = (
|
||||
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS
|
||||
)
|
||||
max_consecutive_transport_failures: int = (
|
||||
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_worker_base_url(self.base_url)
|
||||
_absolute_path(self.bearer_token_file, "Worker bearer token file")
|
||||
_absolute_path(self.work_root, "Worker work root")
|
||||
if isinstance(self.idle_poll_seconds, bool) or not (
|
||||
0.05 <= self.idle_poll_seconds <= 300.0
|
||||
):
|
||||
raise ValueError("Worker idle poll interval is invalid")
|
||||
if isinstance(self.transport_backoff_seconds, bool) or not (
|
||||
0.05 <= self.transport_backoff_seconds <= 300.0
|
||||
):
|
||||
raise ValueError("Worker transport backoff is invalid")
|
||||
if (
|
||||
isinstance(self.max_consecutive_transport_failures, bool)
|
||||
or not isinstance(self.max_consecutive_transport_failures, int)
|
||||
or not 1 <= self.max_consecutive_transport_failures <= 10_000
|
||||
):
|
||||
raise ValueError("Worker transport failure bound is invalid")
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> ObservatoryWorkerServiceConfiguration:
|
||||
values = os.environ if environment is None else environment
|
||||
token_file = _required_path(values, OBSERVATORY_WORKER_TOKEN_FILE_ENV)
|
||||
work_root = _required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV)
|
||||
return cls(
|
||||
base_url=values.get(
|
||||
OBSERVATORY_WORKER_BASE_URL_ENV,
|
||||
DEFAULT_OBSERVATORY_WORKER_BASE_URL,
|
||||
),
|
||||
bearer_token_file=token_file,
|
||||
work_root=work_root,
|
||||
idle_poll_seconds=_environment_float(
|
||||
values,
|
||||
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV,
|
||||
DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS,
|
||||
),
|
||||
transport_backoff_seconds=_environment_float(
|
||||
values,
|
||||
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV,
|
||||
DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS,
|
||||
),
|
||||
max_consecutive_transport_failures=_environment_int(
|
||||
values,
|
||||
OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES_ENV,
|
||||
DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class InstalledObservatoryWorkerService:
|
||||
"""A composed gateway and agent owned by one installed Worker release."""
|
||||
|
||||
configuration: ObservatoryWorkerServiceConfiguration
|
||||
gateway: ObservatoryWorkerHttpGateway
|
||||
agent: ObservatoryWorkerAgent
|
||||
|
||||
def close(self) -> None:
|
||||
self.gateway.close()
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
stop: Event,
|
||||
on_cycle: Callable[[ObservatoryWorkerCycleReport], None] | None = None,
|
||||
) -> None:
|
||||
"""Poll until stopped, with a bounded consecutive transport-failure gate."""
|
||||
|
||||
consecutive_transport_failures = 0
|
||||
try:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
report = self.agent.run_once()
|
||||
except ObservatoryWorkerHttpError as exc:
|
||||
consecutive_transport_failures += 1
|
||||
if (
|
||||
consecutive_transport_failures
|
||||
>= self.configuration.max_consecutive_transport_failures
|
||||
):
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker transport exceeded its consecutive failure bound"
|
||||
) from exc
|
||||
stop.wait(self.configuration.transport_backoff_seconds)
|
||||
continue
|
||||
consecutive_transport_failures = 0
|
||||
if on_cycle is not None:
|
||||
on_cycle(report)
|
||||
if report.state in {
|
||||
"empty",
|
||||
"deferred",
|
||||
"rejected",
|
||||
"lease-lost",
|
||||
}:
|
||||
stop.wait(self.configuration.idle_poll_seconds)
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
|
||||
def compose_installed_observatory_worker_service(
|
||||
*,
|
||||
configuration: ObservatoryWorkerServiceConfiguration,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
executors: ObservatoryWorkerExecutorRegistry,
|
||||
http_transport: httpx.BaseTransport | None = None,
|
||||
) -> InstalledObservatoryWorkerService:
|
||||
"""Bind transport to an install-time registry after exact coverage checks.
|
||||
|
||||
This is the fixed seam a reviewed Worker release calls. There is no
|
||||
dynamic import/provider name in service configuration. Blocked catalog
|
||||
candidates are ignored, while an empty ready set or any missing exact
|
||||
local executor identity rejects service startup before the first claim.
|
||||
"""
|
||||
|
||||
require_ready_executor_coverage(definitions=definitions, executors=executors)
|
||||
bearer_token = load_observatory_worker_bearer_token(
|
||||
configuration.bearer_token_file
|
||||
)
|
||||
try:
|
||||
gateway = ObservatoryWorkerHttpGateway(
|
||||
base_url=configuration.base_url,
|
||||
bearer_token=bearer_token,
|
||||
work_root=configuration.work_root,
|
||||
transport=http_transport,
|
||||
)
|
||||
finally:
|
||||
# The immutable string remains owned by the gateway headers for the
|
||||
# service lifetime; this local binding must not outlive composition.
|
||||
del bearer_token
|
||||
return InstalledObservatoryWorkerService(
|
||||
configuration=configuration,
|
||||
gateway=gateway,
|
||||
agent=ObservatoryWorkerAgent(transport=gateway, executors=executors),
|
||||
)
|
||||
|
||||
|
||||
def require_ready_executor_coverage(
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
executors: ObservatoryWorkerExecutorRegistry,
|
||||
) -> tuple[ObservatoryWorkerExecutorIdentity, ...]:
|
||||
"""Prove every server-advertisable definition has one local identity."""
|
||||
|
||||
ready = definitions.ready_recorded_definitions()
|
||||
if not ready:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"no portable RunDefinition has a sealed ready executor"
|
||||
)
|
||||
identities: list[ObservatoryWorkerExecutorIdentity] = []
|
||||
for definition in ready:
|
||||
identity = ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=definition.executor_release_sha256,
|
||||
image_sha256=definition.executor_image_sha256,
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile_sha256,
|
||||
)
|
||||
try:
|
||||
executors.resolve(identity)
|
||||
except ObservatoryWorkerExecutorUnavailableError as exc:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"a ready portable RunDefinition has no exact local executor identity"
|
||||
) from exc
|
||||
identities.append(identity)
|
||||
return tuple(identities)
|
||||
|
||||
|
||||
def load_observatory_worker_bearer_token(path: Path) -> str:
|
||||
"""Read one private regular ASCII token without accepting links/newlines."""
|
||||
|
||||
candidate = path.expanduser().absolute()
|
||||
descriptor: int | None = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
candidate,
|
||||
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential must be a regular file"
|
||||
)
|
||||
if os.name != "posix":
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"native Worker credential ACL verification is not available; "
|
||||
"use the admitted POSIX Worker service runtime"
|
||||
)
|
||||
if metadata.st_mode & 0o077:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential permissions are too broad"
|
||||
)
|
||||
if not 32 <= metadata.st_size <= 512:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential format is invalid"
|
||||
)
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
descriptor = None
|
||||
payload = stream.read(513)
|
||||
except ObservatoryWorkerServiceError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential is unavailable"
|
||||
) from exc
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
token = payload.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential is not ASCII"
|
||||
) from exc
|
||||
if _TOKEN.fullmatch(token) is None:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"Worker bearer credential format is invalid"
|
||||
)
|
||||
return token
|
||||
|
||||
|
||||
def _validate_worker_base_url(value: str) -> None:
|
||||
parsed = urlsplit(value)
|
||||
if (
|
||||
value != value.strip()
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.path not in {"", "/"}
|
||||
or parsed.scheme not in {"http", "https"}
|
||||
or not parsed.hostname
|
||||
):
|
||||
raise ValueError("Worker Mission Core base URL is invalid")
|
||||
if parsed.scheme == "http" and parsed.hostname not in {
|
||||
"127.0.0.1",
|
||||
"localhost",
|
||||
"::1",
|
||||
}:
|
||||
raise ValueError("unencrypted Worker transport requires a loopback tunnel")
|
||||
|
||||
|
||||
def _absolute_path(path: Path, label: str) -> None:
|
||||
if not path.is_absolute() or str(path) != str(path).strip():
|
||||
raise ValueError(f"{label} must be an absolute path")
|
||||
|
||||
|
||||
def _required_path(environment: Mapping[str, str], name: str) -> Path:
|
||||
value = environment.get(name, "")
|
||||
if not value or value != value.strip():
|
||||
raise ObservatoryWorkerServiceError(f"{name} is required")
|
||||
path = Path(value)
|
||||
_absolute_path(path, name)
|
||||
return path
|
||||
|
||||
|
||||
def _environment_float(
|
||||
environment: Mapping[str, str],
|
||||
name: str,
|
||||
default: float,
|
||||
) -> float:
|
||||
value = environment.get(name, "")
|
||||
if not value:
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError as exc:
|
||||
raise ObservatoryWorkerServiceError(f"{name} is invalid") from exc
|
||||
|
||||
|
||||
def _environment_int(
|
||||
environment: Mapping[str, str],
|
||||
name: str,
|
||||
default: int,
|
||||
) -> int:
|
||||
value = environment.get(name, "")
|
||||
if not value:
|
||||
return default
|
||||
if not value.isascii() or not value.isdecimal():
|
||||
raise ObservatoryWorkerServiceError(f"{name} is invalid")
|
||||
return int(value)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Pure launchd plan for the Mac-owned Worker 006 reverse SSH tunnel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import plistlib
|
||||
import stat
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
OBSERVATORY_WORKER_TUNNEL_LABEL: Final = (
|
||||
"com.nodedc.observatory-worker-tunnel.local"
|
||||
)
|
||||
OBSERVATORY_WORKER_TUNNEL_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-tunnel-plan/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_TUNNEL_PORT: Final = 18080
|
||||
|
||||
|
||||
class ObservatoryWorkerTunnelPlanError(RuntimeError):
|
||||
"""The reverse tunnel cannot be declared without weakening its boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerTunnelLaunchAgentPlan:
|
||||
agent_path: Path
|
||||
data_directory: Path
|
||||
desired_sha256: str
|
||||
desired_payload: bytes
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": OBSERVATORY_WORKER_TUNNEL_SCHEMA,
|
||||
"label": OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||
"agent_path": str(self.agent_path),
|
||||
"data_directory": str(self.data_directory),
|
||||
"desired_sha256": self.desired_sha256,
|
||||
"transport": {
|
||||
"owner": "mac-launchd",
|
||||
"ssh_alias": "mission-gpu",
|
||||
"worker_listener": "127.0.0.1:18080",
|
||||
"mission_core_target": "127.0.0.1:8000",
|
||||
"encrypted": True,
|
||||
"worker_listener_loopback_only": True,
|
||||
"bearer_credential_in_arguments": False,
|
||||
},
|
||||
"changes": {
|
||||
"durable_mutation_performed": False,
|
||||
"requires_hash-gated_install": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def plan_observatory_worker_tunnel_launch_agent(
|
||||
*,
|
||||
data_directory: Path,
|
||||
agent_path: Path,
|
||||
ssh_path: Path = Path("/usr/bin/ssh"),
|
||||
) -> ObservatoryWorkerTunnelLaunchAgentPlan:
|
||||
"""Build, but never install, the exact reverse-loopback declaration."""
|
||||
|
||||
data_root = _private_directory(data_directory)
|
||||
ssh = _exact_executable(ssh_path)
|
||||
target_path = agent_path.expanduser().absolute()
|
||||
log_path = data_root / "observatory-worker-tunnel.log"
|
||||
arguments = [
|
||||
str(ssh),
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"ExitOnForwardFailure=yes",
|
||||
"-o",
|
||||
"ServerAliveInterval=15",
|
||||
"-o",
|
||||
"ServerAliveCountMax=3",
|
||||
"-o",
|
||||
"RequestTTY=no",
|
||||
"-N",
|
||||
"-T",
|
||||
"-R",
|
||||
"127.0.0.1:18080:127.0.0.1:8000",
|
||||
"mission-gpu",
|
||||
]
|
||||
desired = {
|
||||
"Label": OBSERVATORY_WORKER_TUNNEL_LABEL,
|
||||
"ProgramArguments": arguments,
|
||||
"KeepAlive": True,
|
||||
"RunAtLoad": True,
|
||||
"AbandonProcessGroup": False,
|
||||
"ProcessType": "Background",
|
||||
"ThrottleInterval": 5,
|
||||
"ExitTimeOut": 10,
|
||||
"StandardOutPath": str(log_path),
|
||||
"StandardErrorPath": str(log_path),
|
||||
}
|
||||
payload = plistlib.dumps(desired, fmt=plistlib.FMT_XML, sort_keys=True)
|
||||
return ObservatoryWorkerTunnelLaunchAgentPlan(
|
||||
agent_path=target_path,
|
||||
data_directory=data_root,
|
||||
desired_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
desired_payload=payload,
|
||||
)
|
||||
|
||||
|
||||
def _private_directory(path: Path) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ObservatoryWorkerTunnelPlanError(
|
||||
"Mission Core data directory is unavailable"
|
||||
) from exc
|
||||
if (
|
||||
resolved != candidate
|
||||
or not stat.S_ISDIR(metadata.st_mode)
|
||||
or stat.S_IMODE(metadata.st_mode) != 0o700
|
||||
or metadata.st_uid != os.getuid()
|
||||
):
|
||||
raise ObservatoryWorkerTunnelPlanError(
|
||||
"Mission Core data directory is not private and canonical"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _exact_executable(path: Path) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ObservatoryWorkerTunnelPlanError("SSH executable is unavailable") from exc
|
||||
if (
|
||||
resolved != candidate
|
||||
or stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not os.access(candidate, os.X_OK)
|
||||
):
|
||||
raise ObservatoryWorkerTunnelPlanError("SSH executable is not exact")
|
||||
return resolved
|
||||
+204
-31
@@ -47,20 +47,38 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingError,
|
||||
PortableRecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
PortableCalculationProfileRegistry,
|
||||
PortableResultContractValidatorRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
resolve_published_portable_calculation_profile,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
PortableSetupProjector,
|
||||
portable_calculation_profile_registry,
|
||||
)
|
||||
from k1link.observatory.portable_worker_integration import (
|
||||
PortableObservatoryWorkerIntegration,
|
||||
PortableWorkerIntegrationError,
|
||||
PortableWorkerStorageRoots,
|
||||
build_portable_observatory_worker_integration,
|
||||
portable_result_validator_registry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
RecordedRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
@@ -73,6 +91,7 @@ from k1link.sessions import (
|
||||
SessionRecordingPreparationManager,
|
||||
SessionStore,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
from k1link.simulation.projects import SimulationProjectService, SimulationProjectStore
|
||||
from k1link.web.advanced_laboratory_api import build_advanced_laboratory_router
|
||||
from k1link.web.artifact_health_api import build_artifact_health_router
|
||||
@@ -261,6 +280,55 @@ plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
|
||||
OBSERVATORY_PORTABLE_CALCULATION_PROFILES: PortableCalculationProfileRegistry | None
|
||||
OBSERVATORY_PORTABLE_RESULT_VALIDATORS: PortableResultContractValidatorRegistry | None
|
||||
try:
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = PortableRunDefinitionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = portable_calculation_profile_registry(
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
||||
)
|
||||
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = portable_result_validator_registry(
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY
|
||||
)
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = None
|
||||
except (
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableWorkerIntegrationError,
|
||||
OSError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
# Portable definitions are an optional observation-only slice. Registry
|
||||
# drift cannot affect K1, Simulation, legacy LAB, or the exact M49 binding.
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY = None
|
||||
OBSERVATORY_PORTABLE_CALCULATION_PROFILES = None
|
||||
OBSERVATORY_PORTABLE_RESULT_VALIDATORS = None
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR = str(exc)
|
||||
|
||||
|
||||
def _resolve_observatory_calculation_profile(
|
||||
summary: SessionSummary,
|
||||
) -> dict[str, object] | None:
|
||||
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
|
||||
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
|
||||
summary
|
||||
)
|
||||
if legacy is not None:
|
||||
return legacy
|
||||
if (
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
||||
or OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None
|
||||
):
|
||||
return None
|
||||
return resolve_published_portable_calculation_profile(
|
||||
summary,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||
)
|
||||
|
||||
|
||||
def _load_optional_observatory_worker_authentication(
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
||||
@@ -299,9 +367,14 @@ try:
|
||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||
),
|
||||
)
|
||||
recorded_definitions = list(OBSERVATORY_RECORDED_BINDING_SERVICE.definitions.definitions)
|
||||
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is not None:
|
||||
recorded_definitions.extend(
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY.ready_recorded_definitions()
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||
session_store.data_dir,
|
||||
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
|
||||
definitions=RecordedRunDefinitionRegistry(tuple(recorded_definitions)),
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
|
||||
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
|
||||
@@ -316,11 +389,14 @@ OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION_ERROR: str | None
|
||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION = None
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled until claim leases and a verified "
|
||||
"Observatory result publisher are implemented and accepted"
|
||||
(
|
||||
OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
OBSERVATORY_WORKER_AUTHENTICATION_ERROR,
|
||||
) = _load_optional_observatory_worker_authentication(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
token_path=OBSERVATORY_WORKER_TOKEN_PATH,
|
||||
)
|
||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||
@@ -336,37 +412,122 @@ session_recording_materializer = SessionRecordingMaterializer(
|
||||
session_recorded_media_inspector = RecordedMediaInspector(
|
||||
session_store.data_dir / "recorded-media-preparations"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION: PortableObservatoryWorkerIntegration | None
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR: str | None
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS: PortableWorkerStorageRoots | None = None
|
||||
try:
|
||||
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
||||
or "portable definition registry is unavailable"
|
||||
)
|
||||
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable calculation profile registry is unavailable"
|
||||
)
|
||||
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable result validator registry is unavailable"
|
||||
)
|
||||
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
|
||||
or "Observatory recorded-job queue is unavailable"
|
||||
)
|
||||
if session_artifact_gateway is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"central artifact store is not configured"
|
||||
)
|
||||
if session_artifact_gateway.status().central_status != "ready":
|
||||
raise PortableWorkerIntegrationError(
|
||||
"central artifact store is unavailable"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
|
||||
PortableWorkerStorageRoots.from_environment(
|
||||
artifact_store_root=session_artifact_gateway.store.root,
|
||||
)
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
|
||||
build_portable_observatory_worker_integration(
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
artifact_store=session_artifact_gateway.store,
|
||||
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
|
||||
source_cas_root=(
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
|
||||
),
|
||||
result_staging_root=(
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
|
||||
),
|
||||
)
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
|
||||
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||
# Constructing this dormant foundation does not enable the Worker router.
|
||||
# Failure remains isolated from K1, Simulation and legacy LAB.
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled pending sealed installed executors, "
|
||||
"a configured Worker credential, explicit integration acceptance and the "
|
||||
"production gate"
|
||||
+ (
|
||||
""
|
||||
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is None
|
||||
else (
|
||||
"; authentication unavailable: "
|
||||
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||
)
|
||||
)
|
||||
+ (
|
||||
""
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is None
|
||||
else f"; integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||
)
|
||||
)
|
||||
OBSERVATORY_WORKER_DISPATCH_READY = (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
)
|
||||
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
try:
|
||||
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
portable_lab_v1_definition = next(
|
||||
definition
|
||||
for definition in portable_definition_registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
portable_source_capability_service = RecordedK1SourceAdmissionService(
|
||||
if OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None:
|
||||
raise PortableSetupProjectionError(
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR
|
||||
or "portable definition registry is unavailable"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_BINDING_SERVICE = PortableRecordedQueueBindingService(
|
||||
data_dir=session_store.data_dir,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
|
||||
registry=portable_definition_registry,
|
||||
capability_probe=portable_source_capability_service,
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableSetupProjector(
|
||||
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||
except (
|
||||
PortableQueueBindingError,
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableSetupProjectionError,
|
||||
OSError,
|
||||
StopIteration,
|
||||
ValueError,
|
||||
) as exc:
|
||||
# Portable LAB V1 is an optional observation-only slice. A drifted
|
||||
# Portable setup execution is an optional observation-only slice. A drifted
|
||||
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
||||
OBSERVATORY_PORTABLE_BINDING_SERVICE = None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
||||
_ffmpeg = _resolve_media_tool("ffmpeg")
|
||||
@@ -829,6 +990,14 @@ app.include_router(
|
||||
perception_overlay_provider=session_perception_overlay_store,
|
||||
perception_media_provider=session_perception_epoch_store,
|
||||
point_color_renderers=plugin_environment.point_color_renderers,
|
||||
lab_calculation_profile_resolver=(
|
||||
None
|
||||
if (
|
||||
OBSERVATORY_LABORATORY_SETUP_REGISTRY is None
|
||||
and OBSERVATORY_PORTABLE_DEFINITION_REGISTRY is None
|
||||
)
|
||||
else _resolve_observatory_calculation_profile
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
@@ -843,19 +1012,23 @@ app.include_router(
|
||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||
portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||
)
|
||||
)
|
||||
if (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
):
|
||||
if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
assert OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
artifact_transport=(
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
|
||||
),
|
||||
result_publisher=(
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
|
||||
@@ -22,9 +22,19 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingError,
|
||||
PortableQueueBindingIntegrityError,
|
||||
PortableQueueBindingStaleCheckError,
|
||||
PortableRecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionUnavailableError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
PortableSetupProjector,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
@@ -33,6 +43,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
)
|
||||
from k1link.observatory.source_admission import PortableSourceAdmissionError
|
||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
@@ -130,6 +141,8 @@ class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
|
||||
max_length=96,
|
||||
pattern=r"^[a-z][a-z0-9-]{2,95}$",
|
||||
)
|
||||
definition_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
||||
check_sha256: str | None = Field(default=None, pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
def build_observatory_router(
|
||||
@@ -142,8 +155,9 @@ def build_observatory_router(
|
||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||
recorded_job_queue_error: str | None = None,
|
||||
portable_setup_projector: PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector_error: str | None = None,
|
||||
portable_binding_service: PortableRecordedQueueBindingService | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
@@ -214,6 +228,126 @@ def build_observatory_router(
|
||||
available.add(result_id)
|
||||
return frozenset(available)
|
||||
|
||||
def portable_run_preflight(
|
||||
source: SessionSummary,
|
||||
request: ObservatoryRunPreflightRequest,
|
||||
) -> dict[str, object] | None:
|
||||
projector = portable_setup_projector
|
||||
if projector is None or not projector.has_setup(request.setup_id):
|
||||
return None
|
||||
try:
|
||||
projected = projector.project(source, setup_id=request.setup_id)
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||
) from exc
|
||||
definition = projected.get("run_definition")
|
||||
compatibility = projected.get("source_compatibility")
|
||||
executor = projected.get("executor")
|
||||
if (
|
||||
not isinstance(definition, dict)
|
||||
or not isinstance(compatibility, dict)
|
||||
or not isinstance(executor, dict)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||
)
|
||||
expected_digest = definition.get("definition_sha256")
|
||||
if request.definition_sha256 != expected_digest:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Идентичность RunDefinition изменилась; обновите каталог.",
|
||||
)
|
||||
compatible = compatibility.get("compatible") is True
|
||||
executor_ready = executor.get("state") == "ready" and executor.get("ready") is True
|
||||
checked = None
|
||||
check_reason: str | None = None
|
||||
if (
|
||||
compatible
|
||||
and executor_ready
|
||||
and portable_binding_service is not None
|
||||
and recorded_job_queue is not None
|
||||
and isinstance(expected_digest, str)
|
||||
):
|
||||
try:
|
||||
checked = portable_binding_service.check(
|
||||
source_session_id=request.source_session_id,
|
||||
setup_id=request.setup_id,
|
||||
definition_sha256=expected_digest,
|
||||
)
|
||||
except PortableRunDefinitionUnavailableError as exc:
|
||||
check_reason = str(exc)
|
||||
except (PortableQueueBindingError, PortableSourceAdmissionError, ValueError):
|
||||
check_reason = (
|
||||
"Источник или исполняемый portable-релиз не прошёл проверку целостности."
|
||||
)
|
||||
elif compatible and executor_ready:
|
||||
check_reason = "Portable dispatch-контур или durable-очередь недоступны."
|
||||
check_sha256 = None if checked is None else checked.check_sha256
|
||||
queueable = check_sha256 is not None
|
||||
checks: list[dict[str, Any]] = [
|
||||
{
|
||||
"check_id": "source-compatibility",
|
||||
"outcome": "pass" if compatible else "fail",
|
||||
"reason_code": "source-compatible" if compatible else "source-incompatible",
|
||||
"message": str(compatibility.get("reason")),
|
||||
},
|
||||
{
|
||||
"check_id": "executor",
|
||||
"outcome": "pass" if executor_ready else "fail",
|
||||
"reason_code": (
|
||||
"executor-release-sealed"
|
||||
if executor_ready
|
||||
else str(executor.get("reason_code"))
|
||||
),
|
||||
"message": (
|
||||
"Исполняемый portable-релиз и image запечатаны."
|
||||
if executor_ready
|
||||
else str(executor.get("reason"))
|
||||
),
|
||||
},
|
||||
{
|
||||
"check_id": "definition-check",
|
||||
"outcome": "pass" if checked is not None else "fail",
|
||||
"reason_code": (
|
||||
"portable-check-sealed" if checked is not None else "portable-check-unavailable"
|
||||
),
|
||||
"message": (
|
||||
"Источник и RunDefinition связаны одноразовым check SHA."
|
||||
if checked is not None
|
||||
else check_reason
|
||||
or "Portable-проверка недоступна до установки executor-релиза."
|
||||
),
|
||||
},
|
||||
{
|
||||
"check_id": "durable-queue",
|
||||
"outcome": "pass" if queueable else "fail",
|
||||
"reason_code": (
|
||||
"durable-queue-ready" if queueable else "durable-queue-unavailable"
|
||||
),
|
||||
"message": (
|
||||
"Durable-очередь готова принять расчёт по check SHA."
|
||||
if queueable
|
||||
else "Расчёт нельзя поставить в очередь."
|
||||
),
|
||||
},
|
||||
]
|
||||
return {
|
||||
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
|
||||
"source_session_id": request.source_session_id,
|
||||
"setup_id": request.setup_id,
|
||||
"definition_sha256": expected_digest,
|
||||
"check_sha256": check_sha256,
|
||||
"outcome": "queueable" if queueable else "blocked",
|
||||
"submission_allowed": queueable,
|
||||
"checks": checks,
|
||||
"existing_result_ids": [],
|
||||
"executor": executor,
|
||||
"authority": projected.get("authority", dict(_OBSERVATION_ONLY_AUTHORITY)),
|
||||
}
|
||||
|
||||
if portable_setup_projector is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
@@ -230,7 +364,7 @@ def build_observatory_router(
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
|
||||
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||
) from exc
|
||||
|
||||
elif portable_setup_projector_error is not None:
|
||||
@@ -246,10 +380,10 @@ def build_observatory_router(
|
||||
del source_session_id
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 недоступен.",
|
||||
detail="Portable-каталог сетапов недоступен.",
|
||||
)
|
||||
|
||||
if setup_registry is not None:
|
||||
if setup_registry is not None or portable_setup_projector is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/laboratory-setups")
|
||||
def list_observatory_laboratory_setups(
|
||||
@@ -259,6 +393,11 @@ def build_observatory_router(
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
if setup_registry is None:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Каталог legacy-сетапов Обсерватории недоступен.",
|
||||
)
|
||||
source = source_summary(source_session_id)
|
||||
return setup_registry.catalog(
|
||||
source,
|
||||
@@ -270,6 +409,11 @@ def build_observatory_router(
|
||||
request: ObservatoryRunPreflightRequest,
|
||||
) -> dict[str, object]:
|
||||
source = source_summary(request.source_session_id)
|
||||
portable = portable_run_preflight(source, request)
|
||||
if portable is not None:
|
||||
return portable
|
||||
if setup_registry is None:
|
||||
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
|
||||
try:
|
||||
setup_registry.setup(request.setup_id)
|
||||
except KeyError as exc:
|
||||
@@ -639,16 +783,18 @@ def build_observatory_router(
|
||||
detail="Подготовка расчётов Обсерватории недоступна.",
|
||||
)
|
||||
|
||||
if (
|
||||
setup_registry is not None
|
||||
and recorded_binding_service is not None
|
||||
and recorded_job_queue is not None
|
||||
if recorded_job_queue is not None and (
|
||||
recorded_binding_service is not None or portable_binding_service is not None
|
||||
):
|
||||
|
||||
@router.post("/api/v1/observatory/runs", status_code=202)
|
||||
def submit_observatory_recorded_run(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> dict[str, object]:
|
||||
portable_request = (
|
||||
portable_setup_projector is not None
|
||||
and portable_setup_projector.has_setup(request.setup_id)
|
||||
)
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
@@ -662,6 +808,10 @@ def build_observatory_router(
|
||||
if (
|
||||
existing_job.source_session_id != request.source_session_id
|
||||
or existing_job.setup_id != request.setup_id
|
||||
or (
|
||||
portable_request
|
||||
and existing_job.definition_sha256 != request.definition_sha256
|
||||
)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -670,6 +820,113 @@ def build_observatory_router(
|
||||
return existing_job.as_dict()
|
||||
|
||||
source = source_summary(request.source_session_id)
|
||||
if portable_request:
|
||||
if portable_binding_service is None:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable dispatch-контур недоступен.",
|
||||
)
|
||||
if request.definition_sha256 is None or request.check_sha256 is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Для portable-расчёта требуются актуальные definition SHA "
|
||||
"и check SHA из preflight."
|
||||
),
|
||||
)
|
||||
assert portable_setup_projector is not None
|
||||
try:
|
||||
portable_projection = portable_setup_projector.project(
|
||||
source,
|
||||
setup_id=request.setup_id,
|
||||
)
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||
) from exc
|
||||
projected_definition = portable_projection.get("run_definition")
|
||||
projected_executor = portable_projection.get("executor")
|
||||
if not isinstance(projected_definition, dict) or not isinstance(
|
||||
projected_executor, dict
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог сетапов нарушил контракт целостности.",
|
||||
)
|
||||
if projected_definition.get("definition_sha256") != request.definition_sha256:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Идентичность RunDefinition изменилась; повторите preflight.",
|
||||
)
|
||||
if (
|
||||
projected_executor.get("state") != "ready"
|
||||
or projected_executor.get("ready") is not True
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Portable executor-релиз не установлен.",
|
||||
)
|
||||
try:
|
||||
job, _created = portable_binding_service.submit(
|
||||
source_session_id=request.source_session_id,
|
||||
setup_id=request.setup_id,
|
||||
definition_sha256=request.definition_sha256,
|
||||
expected_check_sha256=request.check_sha256,
|
||||
idempotency_key=request.idempotency_key,
|
||||
)
|
||||
return job.as_dict()
|
||||
except PortableQueueBindingStaleCheckError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Источник или RunDefinition изменились; повторите preflight.",
|
||||
) from exc
|
||||
except PortableRunDefinitionUnavailableError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Portable executor-релиз не установлен.",
|
||||
) from exc
|
||||
except (
|
||||
PortableQueueBindingIntegrityError,
|
||||
PortableSourceAdmissionError,
|
||||
) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Portable-привязка источника не прошла проверку целостности.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
try:
|
||||
raced = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
raced = None
|
||||
if (
|
||||
raced is not None
|
||||
and raced.source_session_id == request.source_session_id
|
||||
and raced.setup_id == request.setup_id
|
||||
and raced.definition_sha256 == request.definition_sha256
|
||||
):
|
||||
return raced.as_dict()
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Ключ идемпотентности уже связан с другим расчётом.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueCapacityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Квота durable-очереди расчётов исчерпана.",
|
||||
) from exc
|
||||
except (
|
||||
PortableQueueBindingError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable dispatch-контур недоступен.",
|
||||
) from exc
|
||||
|
||||
if setup_registry is None or recorded_binding_service is None:
|
||||
raise HTTPException(status_code=404, detail="Сетап лаборатории не найден.")
|
||||
try:
|
||||
setup_registry.setup(request.setup_id)
|
||||
except KeyError as exc:
|
||||
|
||||
@@ -18,11 +18,23 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
MAX_RESULT_MANIFEST_BYTES,
|
||||
PortableArtifactTransportError,
|
||||
PortableArtifactTransportIntegrityError,
|
||||
PortableArtifactTransportUnavailableError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedCheckpointError,
|
||||
ObservatoryRecordedJobQueue,
|
||||
@@ -38,6 +50,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
|
||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||
OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1"
|
||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-checkpoint-request/v1"
|
||||
)
|
||||
@@ -46,6 +59,10 @@ OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
|
||||
)
|
||||
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
||||
OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER: Final = "X-Mission-Core-Claim-Token"
|
||||
OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER: Final = (
|
||||
"X-Mission-Core-Claim-Generation"
|
||||
)
|
||||
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
@@ -138,6 +155,13 @@ class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
|
||||
|
||||
class ObservatoryWorkerRenewRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-renew-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
claim_generation: int = Field(ge=1)
|
||||
heartbeat_sequence: int = Field(ge=1)
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
@@ -174,6 +198,8 @@ def build_observatory_worker_router(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
authentication: ObservatoryWorkerAuthentication,
|
||||
artifact_transport: PortableObservatoryArtifactTransport | None = None,
|
||||
result_publisher: PortableObservatoryResultPublisher | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build the bounded Worker pull/state-transition router.
|
||||
|
||||
@@ -182,6 +208,9 @@ def build_observatory_worker_router(
|
||||
hashed, and is compared to the configured digest in constant time.
|
||||
"""
|
||||
|
||||
if result_publisher is not None and artifact_transport is None:
|
||||
raise ValueError("portable result publisher requires artifact transport")
|
||||
|
||||
def require_configured_worker(
|
||||
credentials: Annotated[
|
||||
HTTPAuthorizationCredentials | None,
|
||||
@@ -245,6 +274,20 @@ def build_observatory_worker_router(
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/lease/renew")
|
||||
def renew_job_claim(
|
||||
request: ObservatoryWorkerRenewRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.renew_claim(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
claim_generation=request.claim_generation,
|
||||
heartbeat_sequence=request.heartbeat_sequence,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
@@ -263,14 +306,39 @@ def build_observatory_worker_router(
|
||||
request: ObservatoryWorkerSucceedRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
if artifact_transport is not None:
|
||||
_artifact_call(
|
||||
lambda: artifact_transport.require_completed_for_success(
|
||||
job_id=job_id,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
claim_token=request.claim_token,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
)
|
||||
succeeded = _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
).as_dict()
|
||||
)
|
||||
if artifact_transport is not None and result_publisher is not None:
|
||||
package_root = _artifact_call(
|
||||
lambda: artifact_transport.package_root_for_terminal(succeeded)
|
||||
)
|
||||
try:
|
||||
result_publisher.publish(job=succeeded, package_root=package_root)
|
||||
except PortableResultPublisherError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=(
|
||||
"Recorded result is sealed but its verified publication "
|
||||
"requires reconciliation."
|
||||
),
|
||||
) from exc
|
||||
return succeeded.as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/fail")
|
||||
def fail_job(
|
||||
@@ -286,6 +354,164 @@ def build_observatory_worker_router(
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
if artifact_transport is not None:
|
||||
|
||||
@router.get("/recorded-jobs/{job_id}/source-materialization")
|
||||
def source_materialization(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
claim_token: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
pattern=_CLAIM_TOKEN_PATTERN,
|
||||
),
|
||||
],
|
||||
claim_generation: Annotated[
|
||||
int,
|
||||
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||
],
|
||||
) -> dict[str, object]:
|
||||
return _artifact_call(
|
||||
lambda: artifact_transport.source_manifest(
|
||||
job_id=job_id,
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
.as_dict()
|
||||
)
|
||||
|
||||
@router.get("/recorded-jobs/{job_id}/source-members/{member_id}")
|
||||
def source_member(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||
claim_token: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
pattern=_CLAIM_TOKEN_PATTERN,
|
||||
),
|
||||
],
|
||||
claim_generation: Annotated[
|
||||
int,
|
||||
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||
],
|
||||
) -> FileResponse:
|
||||
member, path = _artifact_call(
|
||||
lambda: artifact_transport.materialize_source_member(
|
||||
job_id=job_id,
|
||||
member_id=member_id,
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
)
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=member.media_type,
|
||||
headers={
|
||||
"ETag": f'"{member.sha256}"',
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"X-Mission-Core-Content-Sha256": member.sha256,
|
||||
},
|
||||
)
|
||||
|
||||
@router.put(
|
||||
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/manifest"
|
||||
)
|
||||
async def stage_result_manifest(
|
||||
request: Request,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||
claim_token: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
pattern=_CLAIM_TOKEN_PATTERN,
|
||||
),
|
||||
],
|
||||
claim_generation: Annotated[
|
||||
int,
|
||||
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||
],
|
||||
) -> dict[str, object]:
|
||||
payload = await _read_bounded_body(request, MAX_RESULT_MANIFEST_BYTES)
|
||||
return _artifact_call(
|
||||
lambda: artifact_transport.stage_result_manifest(
|
||||
job_id=job_id,
|
||||
result_sha256=result_sha256,
|
||||
manifest_payload=payload,
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
.as_dict()
|
||||
)
|
||||
|
||||
@router.put(
|
||||
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/members/{member_id}"
|
||||
)
|
||||
async def upload_result_member(
|
||||
request: Request,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||
member_id: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||
claim_token: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
pattern=_CLAIM_TOKEN_PATTERN,
|
||||
),
|
||||
],
|
||||
claim_generation: Annotated[
|
||||
int,
|
||||
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||
],
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
plan = await artifact_transport.upload_result_member(
|
||||
job_id=job_id,
|
||||
result_sha256=result_sha256,
|
||||
member_id=member_id,
|
||||
chunks=request.stream(),
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
_raise_artifact_or_queue_error(exc)
|
||||
return plan.as_dict()
|
||||
|
||||
@router.post(
|
||||
"/recorded-jobs/{job_id}/result-packages/{result_sha256}/complete"
|
||||
)
|
||||
def complete_result_package(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
result_sha256: Annotated[str, ApiPath(pattern=r"^[a-f0-9]{64}$")],
|
||||
claim_token: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias=OBSERVATORY_WORKER_CLAIM_TOKEN_HEADER,
|
||||
pattern=_CLAIM_TOKEN_PATTERN,
|
||||
),
|
||||
],
|
||||
claim_generation: Annotated[
|
||||
int,
|
||||
Header(alias=OBSERVATORY_WORKER_CLAIM_GENERATION_HEADER, ge=1),
|
||||
],
|
||||
) -> dict[str, object]:
|
||||
return _artifact_call(
|
||||
lambda: artifact_transport.complete_result_upload(
|
||||
job_id=job_id,
|
||||
result_sha256=result_sha256,
|
||||
claim_token=claim_token,
|
||||
claim_generation=claim_generation,
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
.as_dict()
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
@@ -341,3 +567,66 @@ def _queue_call[T](operation: Callable[[], T]) -> T:
|
||||
status_code=503,
|
||||
detail="Recorded-job queue is unavailable.",
|
||||
) from exc
|
||||
|
||||
|
||||
def _artifact_call[T](operation: Callable[[], T]) -> T:
|
||||
try:
|
||||
return _queue_call(operation)
|
||||
except PortableArtifactTransportIntegrityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Worker artifact identity was rejected.",
|
||||
) from exc
|
||||
except PortableArtifactTransportUnavailableError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Worker artifact member is unavailable for this claim.",
|
||||
) from exc
|
||||
except PortableArtifactTransportError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Worker artifact transport is unavailable.",
|
||||
) from exc
|
||||
|
||||
|
||||
def _raise_artifact_or_queue_error(exc: Exception) -> None:
|
||||
if isinstance(exc, PortableArtifactTransportIntegrityError):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Worker artifact identity was rejected.",
|
||||
) from exc
|
||||
if isinstance(exc, PortableArtifactTransportUnavailableError):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Worker artifact member is unavailable for this claim.",
|
||||
) from exc
|
||||
if isinstance(exc, PortableArtifactTransportError):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Worker artifact transport is unavailable.",
|
||||
) from exc
|
||||
_queue_call(lambda: _raise(exc))
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def _raise(exc: Exception) -> None:
|
||||
raise exc
|
||||
|
||||
|
||||
async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes:
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length is not None:
|
||||
try:
|
||||
declared = int(content_length)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="Content-Length is invalid.") from exc
|
||||
if declared < 1 or declared > maximum_bytes:
|
||||
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
|
||||
payload = bytearray()
|
||||
async for chunk in request.stream():
|
||||
payload.extend(chunk)
|
||||
if len(payload) > maximum_bytes:
|
||||
raise HTTPException(status_code=413, detail="Request body is outside bounds.")
|
||||
if not payload:
|
||||
raise HTTPException(status_code=400, detail="Request body is empty.")
|
||||
return bytes(payload)
|
||||
|
||||
@@ -41,6 +41,7 @@ from k1link.sessions.canonical_lab_spatial import (
|
||||
CANONICAL_LAB_SPATIAL_PROFILE,
|
||||
canonical_lab_spatial_frame,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
from k1link.sessions.plugin_contract import RecordedPointColorRenderer
|
||||
from k1link.viewer.recorded import (
|
||||
APPLICATION_ID as RECORDED_APPLICATION_ID,
|
||||
@@ -328,6 +329,9 @@ def build_session_router(
|
||||
perception_overlay_provider: RecordedPerceptionOverlayProvider | None = None,
|
||||
perception_media_provider: RecordedPerceptionMediaProvider | None = None,
|
||||
point_color_renderers: Mapping[str, RecordedPointColorRenderer] | None = None,
|
||||
lab_calculation_profile_resolver: (
|
||||
Callable[[SessionSummary], Mapping[str, object] | None] | None
|
||||
) = None,
|
||||
allow_synchronous_recording_fallback: bool = False,
|
||||
replay_action_id: str = DEFAULT_REPLAY_ACTION_ID,
|
||||
) -> APIRouter:
|
||||
@@ -336,12 +340,29 @@ def build_session_router(
|
||||
router = APIRouter(tags=["observation-sessions"])
|
||||
recorded_media_inspector = media_inspector or RecordedMediaInspector()
|
||||
|
||||
def lab_catalog_document(
|
||||
summary: SessionSummary,
|
||||
contract: Literal["v1", "v2", "v3"],
|
||||
) -> dict[str, Any]:
|
||||
lab = summary.lab
|
||||
if lab is None:
|
||||
raise ValueError("LAB catalog document requires a LAB summary")
|
||||
document = lab.as_dict(include_replay_capability=contract in ("v2", "v3"))
|
||||
if contract == "v3":
|
||||
profile = (
|
||||
None
|
||||
if lab_calculation_profile_resolver is None
|
||||
else lab_calculation_profile_resolver(summary)
|
||||
)
|
||||
document["calculation_profile"] = None if profile is None else dict(profile)
|
||||
return document
|
||||
|
||||
@router.get("/api/v1/observation-sessions")
|
||||
def list_observation_sessions(
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, max_length=128),
|
||||
scope: Literal["all", "source", "laboratory"] = "all",
|
||||
lab_contract: Literal["v1", "v2"] = "v1",
|
||||
lab_contract: Literal["v1", "v2", "v3"] = "v1",
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
_refresh_catalog(catalog_refresher)
|
||||
@@ -349,7 +370,7 @@ def build_session_router(
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
scope=scope,
|
||||
include_capability_projections=lab_contract == "v2",
|
||||
include_capability_projections=lab_contract in ("v2", "v3"),
|
||||
)
|
||||
return {
|
||||
"items": [
|
||||
@@ -364,9 +385,7 @@ def build_session_router(
|
||||
"replayable": item.replayable,
|
||||
**(
|
||||
{
|
||||
"lab": item.lab.as_dict(
|
||||
include_replay_capability=lab_contract == "v2"
|
||||
)
|
||||
"lab": lab_catalog_document(item, lab_contract)
|
||||
}
|
||||
if item.lab is not None
|
||||
else {}
|
||||
|
||||
Reference in New Issue
Block a user