feat(observatory): add installed package dispatch and durable publication
Checkpoint existing backend lifecycle changes. Focused verification found nine legacy fixture failures in portable LAB V1 executor/runtime tests; repair follows separately without rewriting this snapshot. ADR date retains its intentional Markdown hard break.
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,912 @@
|
||||
"""Generic Worker-local launcher for independently installed LAB packages.
|
||||
|
||||
The backend supplies only a sealed job identity. Reviewed local installation
|
||||
bindings translate controller paths to Docker-host paths, while the package
|
||||
owns the exact image, argv, dependency order, resource limits, and mount
|
||||
targets. Every container is a one-shot step and is removed by its exact
|
||||
Docker container id after completion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import stat
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
INSTALLED_LAB_PLAN_PATH,
|
||||
INSTALLED_LAB_RESULT_ROOT,
|
||||
INSTALLED_LAB_SOURCE_ROOT,
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
INSTALLED_LAB_WORK_ROOT,
|
||||
InstalledLabContainer,
|
||||
InstalledLabPackage,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
RESULT_PACKAGE_MANIFEST_NAME,
|
||||
PortableResultPackageIntegrityError,
|
||||
PortableResultPackageManifest,
|
||||
canonical_json,
|
||||
relative_artifact_path,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinition
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerExecutorAdapter,
|
||||
PortableWorkerLocalAssetBinding,
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerRuntimePlan,
|
||||
PortableWorkerSourceStage,
|
||||
inspect_runtime_candidate,
|
||||
)
|
||||
from k1link.observatory.worker_agent import ObservatoryWorkerExecutorRegistration
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from k1link.observatory.worker_service import (
|
||||
ObservatoryWorkerPackageExecutorBuildContext,
|
||||
)
|
||||
|
||||
INSTALLED_LAB_RUN_PLAN_SCHEMA: Final = "missioncore.observatory-installed-lab-run-plan/v1"
|
||||
|
||||
_DOCKER_SOCKET: Final = Path("/var/run/docker.sock")
|
||||
_DOCKER_API_VERSION: Final = "v1.47"
|
||||
_MAX_ENGINE_RESPONSE_BYTES: Final = 1024 * 1024
|
||||
_MAX_RESULT_MANIFEST_BYTES: Final = 1024 * 1024
|
||||
_CONTAINER_ID: Final = re.compile(r"^[a-f0-9]{12,128}$")
|
||||
_ASSET_ID: Final = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_WINDOWS_ABSOLUTE_PATH: Final = re.compile(r"^[A-Za-z]:\\")
|
||||
_ENGINE_PATH_PLACEHOLDER: Final = re.compile(
|
||||
r"(?:\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|"
|
||||
r"\{[A-Za-z_][A-Za-z0-9_]*\})"
|
||||
)
|
||||
|
||||
|
||||
class InstalledLabPackageRunnerError(RuntimeError):
|
||||
"""The generic installed-package runtime changed or failed closed."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabLocalAssetBinding:
|
||||
"""Reviewed local locator; neither path is serialized into a queued job."""
|
||||
|
||||
asset_id: str
|
||||
controller_path: Path | None = None
|
||||
engine_path: str | None = None
|
||||
image_sha256: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _ASSET_ID.fullmatch(self.asset_id) is None:
|
||||
raise ValueError("installed LAB local asset id is invalid")
|
||||
file_binding = self.controller_path is not None or self.engine_path is not None
|
||||
image_binding = self.image_sha256 is not None
|
||||
if file_binding == image_binding:
|
||||
raise ValueError("installed LAB local asset locator is ambiguous")
|
||||
if file_binding:
|
||||
if self.controller_path is None or self.engine_path is None:
|
||||
raise ValueError("installed LAB file asset binding is incomplete")
|
||||
_absolute_controller_path(self.controller_path, "installed LAB asset")
|
||||
_engine_host_path(self.engine_path, "installed LAB asset")
|
||||
elif self.image_sha256 is None or _SHA256.fullmatch(self.image_sha256) is None:
|
||||
raise ValueError("installed LAB image asset digest is invalid")
|
||||
|
||||
def portable_binding(self) -> PortableWorkerLocalAssetBinding:
|
||||
if self.controller_path is not None:
|
||||
return PortableWorkerLocalAssetBinding(
|
||||
asset_id=self.asset_id,
|
||||
file_path=self.controller_path,
|
||||
)
|
||||
return PortableWorkerLocalAssetBinding(
|
||||
asset_id=self.asset_id,
|
||||
image_sha256=self.image_sha256,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabDockerMount:
|
||||
engine_path: str
|
||||
container_path: str
|
||||
read_only: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_engine_host_path(self.engine_path, "installed LAB Docker mount")
|
||||
target = PurePosixPath(self.container_path)
|
||||
if not target.is_absolute() or ".." in target.parts:
|
||||
raise InstalledLabPackageRunnerError("Docker mount target is unsafe")
|
||||
if not isinstance(self.read_only, bool):
|
||||
raise InstalledLabPackageRunnerError("Docker mount mode is invalid")
|
||||
|
||||
def engine_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"Type": "bind",
|
||||
"Source": self.engine_path,
|
||||
"Target": self.container_path,
|
||||
"ReadOnly": self.read_only,
|
||||
"BindOptions": {"Propagation": "rprivate"},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabDockerLaunch:
|
||||
package_id: str
|
||||
container: InstalledLabContainer
|
||||
mounts: tuple[InstalledLabDockerMount, ...]
|
||||
labels: Mapping[str, str]
|
||||
name_token: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
targets = tuple(mount.container_path for mount in self.mounts)
|
||||
if targets != tuple(sorted(targets)) or len(targets) != len(set(targets)):
|
||||
raise InstalledLabPackageRunnerError("Docker mounts are not canonical")
|
||||
writable = tuple(mount for mount in self.mounts if not mount.read_only)
|
||||
if len(writable) != 1 or writable[0].container_path != INSTALLED_LAB_RESULT_ROOT:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"Docker package must expose exactly one writable result mount"
|
||||
)
|
||||
if not re.fullmatch(r"[a-f0-9]{16}", self.name_token):
|
||||
raise InstalledLabPackageRunnerError("Docker launch token is invalid")
|
||||
required_labels = {
|
||||
"com.nodedc.authority",
|
||||
"com.nodedc.component",
|
||||
"com.nodedc.definition-sha256",
|
||||
"com.nodedc.job-id",
|
||||
"com.nodedc.managed-by",
|
||||
"com.nodedc.package-sha256",
|
||||
"com.nodedc.product",
|
||||
"com.nodedc.stack",
|
||||
}
|
||||
if set(self.labels) != required_labels:
|
||||
raise InstalledLabPackageRunnerError("Docker launch label set changed")
|
||||
if (
|
||||
self.labels["com.nodedc.authority"] != "observation-only"
|
||||
or self.labels["com.nodedc.component"] != self.container.container_id
|
||||
or self.labels["com.nodedc.managed-by"] != "mission-core-worker"
|
||||
or self.labels["com.nodedc.product"] != "mission-core"
|
||||
or self.labels["com.nodedc.stack"] != "observatory"
|
||||
):
|
||||
raise InstalledLabPackageRunnerError("Docker launch labels changed")
|
||||
for key in ("com.nodedc.definition-sha256", "com.nodedc.package-sha256"):
|
||||
if _SHA256.fullmatch(self.labels[key]) is None:
|
||||
raise InstalledLabPackageRunnerError("Docker launch digest label is invalid")
|
||||
|
||||
|
||||
class InstalledLabContainerLauncher(Protocol):
|
||||
def __call__(self, launch: InstalledLabDockerLaunch) -> None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DockerEngineInstalledLabLauncher:
|
||||
"""Execute one hardened one-shot package step through the local Engine."""
|
||||
|
||||
socket_path: Path = _DOCKER_SOCKET
|
||||
api_version: str = _DOCKER_API_VERSION
|
||||
transport_factory: Callable[[], httpx.BaseTransport] | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.socket_path.is_absolute():
|
||||
raise InstalledLabPackageRunnerError("Docker socket path is not absolute")
|
||||
if re.fullmatch(r"v[0-9]+\.[0-9]+", self.api_version) is None:
|
||||
raise InstalledLabPackageRunnerError("Docker API version is invalid")
|
||||
|
||||
def __call__(self, launch: InstalledLabDockerLaunch) -> None:
|
||||
if self.transport_factory is None:
|
||||
_require_local_socket(self.socket_path)
|
||||
transport: httpx.BaseTransport = httpx.HTTPTransport(uds=str(self.socket_path))
|
||||
else:
|
||||
transport = self.transport_factory()
|
||||
container_id: str | None = None
|
||||
primary_error: BaseException | None = None
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url="http://docker",
|
||||
transport=transport,
|
||||
timeout=httpx.Timeout(launch.container.timeout_seconds, connect=5.0),
|
||||
) as client:
|
||||
self._verify_image(client, launch.container.image_sha256)
|
||||
container_id = self._create(client, launch)
|
||||
self._empty(
|
||||
client,
|
||||
"POST",
|
||||
self._api_path(f"/containers/{container_id}/start"),
|
||||
{204},
|
||||
)
|
||||
status_code = self._wait(client, container_id)
|
||||
if status_code != 0:
|
||||
log_sha256, log_bytes = self._log_identity(client, container_id)
|
||||
raise InstalledLabPackageRunnerError(
|
||||
f"package step {launch.container.container_id} exited with status "
|
||||
f"{status_code}; logs={log_sha256}:{log_bytes}"
|
||||
)
|
||||
except (httpx.HTTPError, OSError, ValueError) as exc:
|
||||
primary_error = exc
|
||||
raise InstalledLabPackageRunnerError(
|
||||
f"local Docker Engine step {launch.container.container_id} failed"
|
||||
) from exc
|
||||
except BaseException as exc:
|
||||
primary_error = exc
|
||||
raise
|
||||
finally:
|
||||
if container_id is not None:
|
||||
cleanup_error = self._cleanup(container_id)
|
||||
if cleanup_error is not None:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"Docker package container cleanup failed after retry"
|
||||
) from (primary_error or cleanup_error)
|
||||
|
||||
def verify_images(self, image_sha256s: tuple[str, ...]) -> None:
|
||||
"""Prove a canonical installed image inventory without creating a container."""
|
||||
|
||||
if (
|
||||
not image_sha256s
|
||||
or image_sha256s != tuple(sorted(image_sha256s))
|
||||
or len(image_sha256s) != len(set(image_sha256s))
|
||||
or any(_SHA256.fullmatch(value) is None for value in image_sha256s)
|
||||
):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB image inventory is invalid"
|
||||
)
|
||||
if self.transport_factory is None:
|
||||
_require_local_socket(self.socket_path)
|
||||
transport: httpx.BaseTransport = httpx.HTTPTransport(
|
||||
uds=str(self.socket_path)
|
||||
)
|
||||
else:
|
||||
transport = self.transport_factory()
|
||||
with httpx.Client(
|
||||
base_url="http://docker",
|
||||
transport=transport,
|
||||
timeout=httpx.Timeout(10.0, connect=5.0),
|
||||
) as client:
|
||||
for image_sha256 in image_sha256s:
|
||||
self._verify_image(client, image_sha256)
|
||||
|
||||
def _verify_image(self, client: httpx.Client, image_sha256: str) -> None:
|
||||
response = self._response(
|
||||
client,
|
||||
"GET",
|
||||
self._api_path(f"/images/sha256:{image_sha256}/json"),
|
||||
{200},
|
||||
)
|
||||
document = _response_object(response, "Docker image inspection")
|
||||
if document.get("Id") != f"sha256:{image_sha256}":
|
||||
raise InstalledLabPackageRunnerError("Docker image identity changed")
|
||||
|
||||
def _create(self, client: httpx.Client, launch: InstalledLabDockerLaunch) -> str:
|
||||
component = launch.container.container_id[:32]
|
||||
name = f"ndc-observatory-{component}-{launch.name_token}"
|
||||
response = self._response(
|
||||
client,
|
||||
"POST",
|
||||
self._api_path(f"/containers/create?name={name}"),
|
||||
{201},
|
||||
json_body=_container_create_document(launch),
|
||||
)
|
||||
document = _response_object(response, "Docker container creation")
|
||||
container_id = document.get("Id")
|
||||
if not isinstance(container_id, str) or _CONTAINER_ID.fullmatch(container_id) is None:
|
||||
raise InstalledLabPackageRunnerError("Docker container id is invalid")
|
||||
if document.get("Warnings") not in (None, []):
|
||||
raise InstalledLabPackageRunnerError("Docker container creation returned warnings")
|
||||
return container_id
|
||||
|
||||
def _cleanup(self, container_id: str) -> BaseException | None:
|
||||
last_error: BaseException | None = None
|
||||
for _attempt in range(2):
|
||||
try:
|
||||
transport: httpx.BaseTransport = (
|
||||
httpx.HTTPTransport(uds=str(self.socket_path))
|
||||
if self.transport_factory is None
|
||||
else self.transport_factory()
|
||||
)
|
||||
with httpx.Client(
|
||||
base_url="http://docker",
|
||||
transport=transport,
|
||||
timeout=httpx.Timeout(10.0, connect=5.0),
|
||||
) as client:
|
||||
self._empty(
|
||||
client,
|
||||
"DELETE",
|
||||
self._api_path(f"/containers/{container_id}?force=1&v=1"),
|
||||
{204, 404},
|
||||
)
|
||||
return None
|
||||
except (httpx.HTTPError, OSError, ValueError, InstalledLabPackageRunnerError) as exc:
|
||||
last_error = exc
|
||||
return last_error
|
||||
|
||||
def _wait(self, client: httpx.Client, container_id: str) -> int:
|
||||
response = self._response(
|
||||
client,
|
||||
"POST",
|
||||
self._api_path(f"/containers/{container_id}/wait?condition=not-running"),
|
||||
{200},
|
||||
)
|
||||
document = _response_object(response, "Docker container wait")
|
||||
status_code = document.get("StatusCode")
|
||||
if isinstance(status_code, bool) or not isinstance(status_code, int):
|
||||
raise InstalledLabPackageRunnerError("Docker exit status is invalid")
|
||||
if document.get("Error") not in (None, {"Message": ""}):
|
||||
raise InstalledLabPackageRunnerError("Docker wait returned an Engine error")
|
||||
return status_code
|
||||
|
||||
def _log_identity(self, client: httpx.Client, container_id: str) -> tuple[str, int]:
|
||||
response = self._response(
|
||||
client,
|
||||
"GET",
|
||||
self._api_path(f"/containers/{container_id}/logs?stdout=1&stderr=1&tail=200"),
|
||||
{200},
|
||||
)
|
||||
return hashlib.sha256(response.content).hexdigest(), len(response.content)
|
||||
|
||||
def _api_path(self, path: str) -> str:
|
||||
return f"/{self.api_version}{path}"
|
||||
|
||||
@staticmethod
|
||||
def _empty(
|
||||
client: httpx.Client,
|
||||
method: str,
|
||||
path: str,
|
||||
statuses: set[int],
|
||||
) -> None:
|
||||
DockerEngineInstalledLabLauncher._response(client, method, path, statuses)
|
||||
|
||||
@staticmethod
|
||||
def _response(
|
||||
client: httpx.Client,
|
||||
method: str,
|
||||
path: str,
|
||||
statuses: set[int],
|
||||
*,
|
||||
json_body: Mapping[str, object] | None = None,
|
||||
) -> httpx.Response:
|
||||
with client.stream(method, path, json=json_body) as streamed:
|
||||
if streamed.status_code not in statuses:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
f"Docker Engine rejected {method} {path.split('?')[0]} "
|
||||
f"with status {streamed.status_code}"
|
||||
)
|
||||
declared_length = streamed.headers.get("content-length")
|
||||
if declared_length is not None:
|
||||
try:
|
||||
length = int(declared_length)
|
||||
except ValueError as exc:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"Docker Engine response length is invalid"
|
||||
) from exc
|
||||
if length < 0 or length > _MAX_ENGINE_RESPONSE_BYTES:
|
||||
raise InstalledLabPackageRunnerError("Docker Engine response is too large")
|
||||
payload = bytearray()
|
||||
blocks = (
|
||||
(streamed.content,)
|
||||
if streamed.is_stream_consumed
|
||||
else streamed.iter_raw(chunk_size=16 * 1024)
|
||||
)
|
||||
for block in blocks:
|
||||
if len(payload) + len(block) > _MAX_ENGINE_RESPONSE_BYTES:
|
||||
raise InstalledLabPackageRunnerError("Docker Engine response is too large")
|
||||
payload.extend(block)
|
||||
return httpx.Response(
|
||||
status_code=streamed.status_code,
|
||||
headers=streamed.headers,
|
||||
content=bytes(payload),
|
||||
request=streamed.request,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabPackageProfileRunner:
|
||||
package: InstalledLabPackage
|
||||
definition: PortableRunDefinition
|
||||
controller_work_root: Path
|
||||
engine_work_root: str
|
||||
local_assets: tuple[InstalledLabLocalAssetBinding, ...]
|
||||
launcher: InstalledLabContainerLauncher
|
||||
token_factory: Callable[[], str] = lambda: secrets.token_hex(8)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_absolute_controller_path(self.controller_work_root, "Worker package root")
|
||||
_engine_host_path(self.engine_work_root, "Worker package root")
|
||||
asset_ids = tuple(binding.asset_id for binding in self.local_assets)
|
||||
if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB local assets must be canonical and unique"
|
||||
)
|
||||
|
||||
def run(
|
||||
self,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
source: PortableWorkerSourceStage,
|
||||
) -> PortableWorkerResultDraft:
|
||||
if (
|
||||
plan.setup_id != self.package.setup_id
|
||||
or plan.definition_sha256 != self.package.definition_sha256
|
||||
or plan.candidate_sha256 != self.package.runtime_candidate_sha256
|
||||
or plan.result_contract_sha256 != self.package.result_contract_sha256
|
||||
or plan.source_bundle_sha256 != source.source_bundle_sha256
|
||||
or plan.source_capability_manifest_sha256 != source.source_capability_manifest_sha256
|
||||
):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB run plan differs from its package or source"
|
||||
)
|
||||
controller_root = _real_directory(
|
||||
self.controller_work_root,
|
||||
"Worker package root",
|
||||
)
|
||||
source_root = _real_directory(source.root, "installed LAB source root")
|
||||
_require_descendant(source_root, controller_root, "installed LAB source root")
|
||||
jobs_root = controller_root / "jobs"
|
||||
jobs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
attempt_token = self.token_factory()
|
||||
if re.fullmatch(r"[a-f0-9]{16}", attempt_token) is None:
|
||||
raise InstalledLabPackageRunnerError("installed LAB attempt token is invalid")
|
||||
job_root = jobs_root / plan.job_id / attempt_token
|
||||
try:
|
||||
job_root.mkdir(mode=0o700, parents=True, exist_ok=False)
|
||||
output_root = job_root / "result-staging"
|
||||
output_root.mkdir(mode=0o700)
|
||||
steps_root = job_root / "steps"
|
||||
steps_root.mkdir(mode=0o700)
|
||||
plan_path = job_root / "run-plan.json"
|
||||
_write_exclusive(
|
||||
plan_path,
|
||||
canonical_json(
|
||||
{
|
||||
"schema_version": INSTALLED_LAB_RUN_PLAN_SCHEMA,
|
||||
"runtime_plan": plan.as_dict(),
|
||||
"package_id": self.package.package_id,
|
||||
"package_sha256": self.package.package_sha256,
|
||||
"authority": self.definition.authority.as_dict(),
|
||||
}
|
||||
),
|
||||
)
|
||||
for container in _topological_containers(self.package):
|
||||
container_output_root = output_root
|
||||
if container.role == "step":
|
||||
container_output_root = steps_root / container.container_id
|
||||
container_output_root.mkdir(mode=0o700)
|
||||
self.launcher(
|
||||
self._launch(
|
||||
container=container,
|
||||
plan=plan,
|
||||
source_root=source_root,
|
||||
plan_path=plan_path,
|
||||
output_root=container_output_root,
|
||||
steps_root=steps_root,
|
||||
name_token=attempt_token,
|
||||
)
|
||||
)
|
||||
return _read_result_draft(
|
||||
output_root,
|
||||
plan=plan,
|
||||
definition=self.definition,
|
||||
)
|
||||
except FileExistsError as exc:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB attempt directory already exists"
|
||||
) from exc
|
||||
except OSError as exc:
|
||||
shutil.rmtree(job_root, ignore_errors=True)
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB job workspace is unavailable"
|
||||
) from exc
|
||||
except BaseException:
|
||||
shutil.rmtree(job_root, ignore_errors=True)
|
||||
raise
|
||||
|
||||
def _launch(
|
||||
self,
|
||||
*,
|
||||
container: InstalledLabContainer,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
source_root: Path,
|
||||
plan_path: Path,
|
||||
output_root: Path,
|
||||
steps_root: Path,
|
||||
name_token: str,
|
||||
) -> InstalledLabDockerLaunch:
|
||||
by_asset = {binding.asset_id: binding for binding in self.local_assets}
|
||||
mounts = [
|
||||
InstalledLabDockerMount(
|
||||
_translate_work_path(
|
||||
source_root,
|
||||
controller_root=self.controller_work_root,
|
||||
engine_root=self.engine_work_root,
|
||||
),
|
||||
INSTALLED_LAB_SOURCE_ROOT,
|
||||
True,
|
||||
),
|
||||
InstalledLabDockerMount(
|
||||
_translate_work_path(
|
||||
plan_path,
|
||||
controller_root=self.controller_work_root,
|
||||
engine_root=self.engine_work_root,
|
||||
),
|
||||
INSTALLED_LAB_PLAN_PATH,
|
||||
True,
|
||||
),
|
||||
InstalledLabDockerMount(
|
||||
_translate_work_path(
|
||||
output_root,
|
||||
controller_root=self.controller_work_root,
|
||||
engine_root=self.engine_work_root,
|
||||
),
|
||||
INSTALLED_LAB_RESULT_ROOT,
|
||||
False,
|
||||
),
|
||||
]
|
||||
for dependency in _container_ancestors(self.package, container.container_id):
|
||||
dependency_root = _real_directory(
|
||||
steps_root / dependency,
|
||||
"installed LAB dependency output",
|
||||
)
|
||||
mounts.append(
|
||||
InstalledLabDockerMount(
|
||||
_translate_work_path(
|
||||
dependency_root,
|
||||
controller_root=self.controller_work_root,
|
||||
engine_root=self.engine_work_root,
|
||||
),
|
||||
f"{INSTALLED_LAB_STEP_INPUT_ROOT}/{dependency}",
|
||||
True,
|
||||
)
|
||||
)
|
||||
for package_mount in container.mounts:
|
||||
binding = by_asset.get(package_mount.asset_id)
|
||||
if binding is None or binding.engine_path is None:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"package file mount has no reviewed local binding"
|
||||
)
|
||||
mounts.append(
|
||||
InstalledLabDockerMount(
|
||||
binding.engine_path,
|
||||
package_mount.target,
|
||||
True,
|
||||
)
|
||||
)
|
||||
return InstalledLabDockerLaunch(
|
||||
package_id=self.package.package_id,
|
||||
container=container,
|
||||
mounts=tuple(sorted(mounts, key=lambda item: item.container_path)),
|
||||
labels={
|
||||
"com.nodedc.authority": "observation-only",
|
||||
"com.nodedc.component": container.container_id,
|
||||
"com.nodedc.definition-sha256": plan.definition_sha256,
|
||||
"com.nodedc.job-id": plan.job_id,
|
||||
"com.nodedc.managed-by": "mission-core-worker",
|
||||
"com.nodedc.package-sha256": self.package.package_sha256,
|
||||
"com.nodedc.product": "mission-core",
|
||||
"com.nodedc.stack": "observatory",
|
||||
},
|
||||
name_token=name_token,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabPackageExecutorFactory:
|
||||
"""One generic factory for every independently installed package."""
|
||||
|
||||
local_assets: tuple[InstalledLabLocalAssetBinding, ...]
|
||||
engine_work_root: str
|
||||
launcher: InstalledLabContainerLauncher = DockerEngineInstalledLabLauncher()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
asset_ids = tuple(binding.asset_id for binding in self.local_assets)
|
||||
if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB factory assets must be canonical and unique"
|
||||
)
|
||||
_engine_host_path(self.engine_work_root, "Worker Engine work root")
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
context: ObservatoryWorkerPackageExecutorBuildContext,
|
||||
) -> ObservatoryWorkerExecutorRegistration:
|
||||
context.package.bind(context.definition, context.candidate)
|
||||
by_asset = {binding.asset_id: binding for binding in self.local_assets}
|
||||
package_assets = tuple(
|
||||
by_asset[asset_id] for asset_id in context.package.asset_ids if asset_id in by_asset
|
||||
)
|
||||
if tuple(binding.asset_id for binding in package_assets) != context.package.asset_ids:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB package has an incomplete local asset map"
|
||||
)
|
||||
admission = inspect_runtime_candidate(
|
||||
context.candidate,
|
||||
{binding.asset_id: binding.portable_binding() for binding in package_assets},
|
||||
)
|
||||
if not admission.ready:
|
||||
blockers = ",".join(admission.blockers)
|
||||
raise InstalledLabPackageRunnerError(
|
||||
f"installed LAB package local assets are not admitted: {blockers}"
|
||||
)
|
||||
runner = InstalledLabPackageProfileRunner(
|
||||
package=context.package,
|
||||
definition=context.definition,
|
||||
controller_work_root=context.work_root,
|
||||
engine_work_root=self.engine_work_root,
|
||||
local_assets=package_assets,
|
||||
launcher=self.launcher,
|
||||
)
|
||||
adapter = PortableWorkerExecutorAdapter(
|
||||
candidate=context.candidate,
|
||||
definition=context.definition,
|
||||
admission=admission,
|
||||
source_materializer=context.source_transport,
|
||||
runner=runner,
|
||||
publisher=context.result_transport,
|
||||
)
|
||||
return ObservatoryWorkerExecutorRegistration(
|
||||
identity=context.package.executor_identity,
|
||||
adapter=adapter,
|
||||
)
|
||||
|
||||
|
||||
def _topological_containers(
|
||||
package: InstalledLabPackage,
|
||||
) -> tuple[InstalledLabContainer, ...]:
|
||||
remaining = {container.container_id: container for container in package.containers}
|
||||
completed: set[str] = set()
|
||||
ordered: list[InstalledLabContainer] = []
|
||||
while remaining:
|
||||
ready = tuple(
|
||||
container
|
||||
for container in remaining.values()
|
||||
if set(container.depends_on).issubset(completed)
|
||||
)
|
||||
if not ready:
|
||||
raise InstalledLabPackageRunnerError("package container topology is blocked")
|
||||
for container in sorted(ready, key=lambda item: item.container_id):
|
||||
ordered.append(container)
|
||||
completed.add(container.container_id)
|
||||
del remaining[container.container_id]
|
||||
return tuple(ordered)
|
||||
|
||||
|
||||
def _container_ancestors(
|
||||
package: InstalledLabPackage,
|
||||
container_id: str,
|
||||
) -> tuple[str, ...]:
|
||||
by_id = {container.container_id: container for container in package.containers}
|
||||
ancestors: set[str] = set()
|
||||
|
||||
def collect(current: str) -> None:
|
||||
for dependency in by_id[current].depends_on:
|
||||
if dependency not in ancestors:
|
||||
ancestors.add(dependency)
|
||||
collect(dependency)
|
||||
|
||||
collect(container_id)
|
||||
return tuple(sorted(ancestors))
|
||||
|
||||
|
||||
def _container_create_document(launch: InstalledLabDockerLaunch) -> dict[str, object]:
|
||||
container = launch.container
|
||||
device_requests: list[dict[str, object]] = []
|
||||
if container.gpu_count:
|
||||
device_requests.append(
|
||||
{
|
||||
"Driver": "nvidia",
|
||||
"Count": container.gpu_count,
|
||||
"Capabilities": [["gpu"]],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"Image": f"sha256:{container.image_sha256}",
|
||||
"Cmd": list(container.argv),
|
||||
"WorkingDir": INSTALLED_LAB_WORK_ROOT,
|
||||
"Env": [
|
||||
"HF_HUB_OFFLINE=1",
|
||||
"TRANSFORMERS_OFFLINE=1",
|
||||
"PYTHONDONTWRITEBYTECODE=1",
|
||||
],
|
||||
"Labels": dict(sorted(launch.labels.items())),
|
||||
"NetworkDisabled": True,
|
||||
"OpenStdin": False,
|
||||
"StdinOnce": False,
|
||||
"Tty": False,
|
||||
"AttachStdout": True,
|
||||
"AttachStderr": True,
|
||||
"HostConfig": {
|
||||
"AutoRemove": False,
|
||||
"CapDrop": ["ALL"],
|
||||
"DeviceRequests": device_requests,
|
||||
"Init": True,
|
||||
"IpcMode": "private",
|
||||
"Memory": container.memory_bytes,
|
||||
"MemorySwap": container.memory_bytes,
|
||||
"Mounts": [mount.engine_document() for mount in launch.mounts],
|
||||
"NanoCpus": container.nano_cpus,
|
||||
"NetworkMode": "none",
|
||||
"PidsLimit": container.pids_limit,
|
||||
"Privileged": False,
|
||||
"ReadonlyRootfs": True,
|
||||
"SecurityOpt": ["no-new-privileges:true"],
|
||||
"ShmSize": container.shm_bytes,
|
||||
"Tmpfs": {
|
||||
INSTALLED_LAB_WORK_ROOT: (
|
||||
f"rw,noexec,nosuid,nodev,size={container.tmpfs_bytes},mode=1777"
|
||||
),
|
||||
"/tmp": "rw,noexec,nosuid,nodev,size=67108864,mode=1777",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _read_result_draft(
|
||||
root: Path,
|
||||
*,
|
||||
plan: PortableWorkerRuntimePlan,
|
||||
definition: PortableRunDefinition,
|
||||
) -> PortableWorkerResultDraft:
|
||||
result_root = _real_directory(root, "installed LAB result root")
|
||||
manifest_path = result_root / RESULT_PACKAGE_MANIFEST_NAME
|
||||
try:
|
||||
metadata = manifest_path.lstat()
|
||||
if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB result manifest is not a regular file"
|
||||
)
|
||||
if not 0 < metadata.st_size <= _MAX_RESULT_MANIFEST_BYTES:
|
||||
raise InstalledLabPackageRunnerError("installed LAB result manifest size is invalid")
|
||||
payload = manifest_path.read_bytes()
|
||||
manifest = PortableResultPackageManifest.from_bytes(payload)
|
||||
except InstalledLabPackageRunnerError:
|
||||
raise
|
||||
except (OSError, PortableResultPackageIntegrityError) as exc:
|
||||
raise InstalledLabPackageRunnerError("installed LAB result manifest is invalid") from exc
|
||||
source = manifest.source
|
||||
run_definition = manifest.run_definition
|
||||
result = manifest.result
|
||||
if (
|
||||
manifest.job.get("job_id") != plan.job_id
|
||||
or source.get("bundle_sha256") != plan.source_bundle_sha256
|
||||
or source.get("capability_manifest_sha256") != plan.source_capability_manifest_sha256
|
||||
or run_definition.get("definition_sha256") != plan.definition_sha256
|
||||
or result.get("result_contract_sha256") != plan.result_contract_sha256
|
||||
or result.get("result_schema") != definition.result_contract.result_schema
|
||||
or result.get("result_kind") != definition.result_contract.result_kind
|
||||
or manifest.authority != definition.authority.as_dict()
|
||||
):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB result package lost its sealed identity"
|
||||
)
|
||||
result_id = result.get("result_id")
|
||||
if not isinstance(result_id, str):
|
||||
raise InstalledLabPackageRunnerError("installed LAB result id is invalid")
|
||||
expected_files = {RESULT_PACKAGE_MANIFEST_NAME}
|
||||
for artifact in manifest.artifacts:
|
||||
relative = relative_artifact_path(artifact.relative_path)
|
||||
artifact_path = result_root.joinpath(*relative.parts)
|
||||
try:
|
||||
metadata = artifact_path.lstat()
|
||||
if not stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB result artifact is not a regular file"
|
||||
)
|
||||
if metadata.st_size != artifact.byte_length:
|
||||
raise InstalledLabPackageRunnerError("installed LAB result artifact size changed")
|
||||
if _sha256_file(artifact_path) != artifact.sha256:
|
||||
raise InstalledLabPackageRunnerError("installed LAB result artifact digest changed")
|
||||
except OSError as exc:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB result artifact is unavailable"
|
||||
) from exc
|
||||
expected_files.add(relative.as_posix())
|
||||
actual_files: set[str] = set()
|
||||
for candidate in result_root.rglob("*"):
|
||||
relative_name = candidate.relative_to(result_root).as_posix()
|
||||
metadata = candidate.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError("installed LAB result contains a link")
|
||||
if stat.S_ISREG(metadata.st_mode):
|
||||
actual_files.add(relative_name)
|
||||
elif not stat.S_ISDIR(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError("installed LAB result contains a special file")
|
||||
if actual_files != expected_files:
|
||||
raise InstalledLabPackageRunnerError("installed LAB result contains undeclared files")
|
||||
return PortableWorkerResultDraft(
|
||||
root=result_root,
|
||||
result_id=result_id,
|
||||
result_sha256=manifest.manifest_sha256,
|
||||
result_contract_sha256=plan.result_contract_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _translate_work_path(
|
||||
path: Path,
|
||||
*,
|
||||
controller_root: Path,
|
||||
engine_root: str,
|
||||
) -> str:
|
||||
candidate = path.expanduser().absolute()
|
||||
root = controller_root.expanduser().absolute()
|
||||
try:
|
||||
relative = candidate.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"Worker path is outside the Docker-host work binding"
|
||||
) from exc
|
||||
if _WINDOWS_ABSOLUTE_PATH.match(engine_root):
|
||||
suffix = "\\".join(relative.parts)
|
||||
return engine_root.rstrip("\\") + (f"\\{suffix}" if suffix else "")
|
||||
suffix = "/".join(relative.parts)
|
||||
return engine_root.rstrip("/") + (f"/{suffix}" if suffix else "")
|
||||
|
||||
|
||||
def _require_descendant(path: Path, root: Path, label: str) -> None:
|
||||
if path == root or not path.is_relative_to(root):
|
||||
raise InstalledLabPackageRunnerError(f"{label} is outside the Worker package root")
|
||||
|
||||
|
||||
def _absolute_controller_path(path: Path, label: str) -> None:
|
||||
if not path.is_absolute():
|
||||
raise ValueError(f"{label} controller path must be absolute")
|
||||
|
||||
|
||||
def _engine_host_path(value: str, label: str) -> None:
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not value
|
||||
or value != value.strip()
|
||||
or "\x00" in value
|
||||
or "\n" in value
|
||||
or "\r" in value
|
||||
or _ENGINE_PATH_PLACEHOLDER.search(value) is not None
|
||||
or ".." in PurePosixPath(value.replace("\\", "/")).parts
|
||||
or not (value.startswith("/") or _WINDOWS_ABSOLUTE_PATH.match(value))
|
||||
):
|
||||
raise ValueError(f"{label} Engine path is invalid")
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise InstalledLabPackageRunnerError(f"{label} is unavailable") from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError(f"{label} is not a real directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _write_exclusive(path: Path, payload: bytes) -> None:
|
||||
with path.open("xb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _response_object(response: httpx.Response, label: str) -> dict[str, object]:
|
||||
try:
|
||||
document: object = response.json()
|
||||
except ValueError as exc:
|
||||
raise InstalledLabPackageRunnerError(f"{label} is not JSON") from exc
|
||||
if not isinstance(document, dict) or any(not isinstance(key, str) for key in document):
|
||||
raise InstalledLabPackageRunnerError(f"{label} is not an object")
|
||||
return document
|
||||
|
||||
|
||||
def _require_local_socket(path: Path) -> None:
|
||||
try:
|
||||
metadata = path.stat()
|
||||
except OSError as exc:
|
||||
raise InstalledLabPackageRunnerError("local Docker socket is unavailable") from exc
|
||||
if not stat.S_ISSOCK(metadata.st_mode):
|
||||
raise InstalledLabPackageRunnerError("local Docker socket is not a socket")
|
||||
@@ -0,0 +1,721 @@
|
||||
"""Worker-local, content-addressed execution packages for portable LABs.
|
||||
|
||||
Mission Core seals data identities; it never sends executable instructions.
|
||||
This module defines the separately installed Worker package that is allowed to
|
||||
contain reviewed container argv and in-container mount targets. A package is
|
||||
bound to one exact RunDefinition and runtime candidate and uses one stable I/O
|
||||
surface, so adding a conforming LAB does not require another Worker micro-app.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Final, Literal
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerRuntimeCandidate,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import RecordedExecutorIdentity
|
||||
|
||||
INSTALLED_LAB_PACKAGE_SCHEMA: Final = "missioncore.observatory-installed-lab-package/v1"
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA: Final = (
|
||||
"missioncore.observatory-installed-lab-package-registry/v1"
|
||||
)
|
||||
INSTALLED_LAB_CONTAINER_IO_SCHEMA: Final = "missioncore.observatory-installed-lab-container-io/v2"
|
||||
|
||||
INSTALLED_LAB_SOURCE_ROOT: Final = "/missioncore/input/source"
|
||||
INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/input/run-plan.json"
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT: Final = "/missioncore/input/steps"
|
||||
INSTALLED_LAB_RESULT_ROOT: Final = "/missioncore/output"
|
||||
INSTALLED_LAB_WORK_ROOT: Final = "/missioncore/work"
|
||||
|
||||
_MAX_REGISTRY_BYTES: Final = 1024 * 1024
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_ASSET_ID = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_FORBIDDEN_KEYS = frozenset(
|
||||
{
|
||||
"host_path",
|
||||
"host-path",
|
||||
"hostpath",
|
||||
"secret",
|
||||
"secrets",
|
||||
"token",
|
||||
"password",
|
||||
"privileged",
|
||||
"docker_socket",
|
||||
}
|
||||
)
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
type InstalledLabExecutionMode = Literal["single-container", "fixed-stack"]
|
||||
type InstalledLabContainerRole = Literal["step", "result-writer"]
|
||||
type InstalledLabNetworkMode = Literal["none"]
|
||||
|
||||
|
||||
class InstalledLabPackageError(RuntimeError):
|
||||
"""An installed LAB package is malformed or not exactly bound."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabPackageMount:
|
||||
"""Read-only asset mount; its Worker-local source path is not serialized."""
|
||||
|
||||
asset_id: str
|
||||
target: str
|
||||
read_only: Literal[True] = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.asset_id, _ASSET_ID, "package mount asset id")
|
||||
target = PurePosixPath(self.target)
|
||||
reserved_roots = tuple(
|
||||
PurePosixPath(value)
|
||||
for value in (
|
||||
INSTALLED_LAB_SOURCE_ROOT,
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
INSTALLED_LAB_RESULT_ROOT,
|
||||
INSTALLED_LAB_WORK_ROOT,
|
||||
)
|
||||
)
|
||||
if (
|
||||
not target.is_absolute()
|
||||
or ".." in target.parts
|
||||
or target == PurePosixPath(INSTALLED_LAB_PLAN_PATH)
|
||||
or any(target == root or target.is_relative_to(root) for root in reserved_roots)
|
||||
):
|
||||
raise InstalledLabPackageError("package asset mount target is unsafe")
|
||||
if self.read_only is not True:
|
||||
raise InstalledLabPackageError("package assets must remain read-only")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"asset_id": self.asset_id,
|
||||
"target": self.target,
|
||||
"read_only": True,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabContainer:
|
||||
container_id: str
|
||||
role: InstalledLabContainerRole
|
||||
image_sha256: str
|
||||
argv: tuple[str, ...]
|
||||
depends_on: tuple[str, ...]
|
||||
mounts: tuple[InstalledLabPackageMount, ...]
|
||||
network: InstalledLabNetworkMode
|
||||
gpu_count: int
|
||||
memory_bytes: int
|
||||
nano_cpus: int
|
||||
pids_limit: int
|
||||
shm_bytes: int
|
||||
tmpfs_bytes: int
|
||||
timeout_seconds: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.container_id, _IDENTIFIER, "package container id")
|
||||
_digest(self.image_sha256, "package container image sha256")
|
||||
if self.role not in ("step", "result-writer"):
|
||||
raise InstalledLabPackageError("package container role is invalid")
|
||||
if not self.argv or len(self.argv) > 64:
|
||||
raise InstalledLabPackageError("package container argv is invalid")
|
||||
for argument in self.argv:
|
||||
if (
|
||||
not isinstance(argument, str)
|
||||
or not argument
|
||||
or len(argument) > 1_024
|
||||
or "\x00" in argument
|
||||
or "\n" in argument
|
||||
or "\r" in argument
|
||||
or "${" in argument
|
||||
):
|
||||
raise InstalledLabPackageError("package container argument is unsafe")
|
||||
if self.depends_on != tuple(sorted(self.depends_on)) or len(self.depends_on) != len(
|
||||
set(self.depends_on)
|
||||
):
|
||||
raise InstalledLabPackageError("package container dependencies must be canonical")
|
||||
for dependency in self.depends_on:
|
||||
_pattern(dependency, _IDENTIFIER, "package container dependency")
|
||||
mount_keys = [(mount.target, mount.asset_id) for mount in self.mounts]
|
||||
if mount_keys != sorted(mount_keys) or len(mount_keys) != len(set(mount_keys)):
|
||||
raise InstalledLabPackageError("package mounts must be canonical and unique")
|
||||
if self.network != "none":
|
||||
raise InstalledLabPackageError("package container network mode is invalid")
|
||||
if isinstance(self.gpu_count, bool) or not 0 <= self.gpu_count <= 8:
|
||||
raise InstalledLabPackageError("package container GPU count is invalid")
|
||||
for value, minimum, maximum, label in (
|
||||
(self.memory_bytes, 64 * 1024**2, 512 * 1024**3, "memory"),
|
||||
(self.nano_cpus, 100_000_000, 128_000_000_000, "CPU"),
|
||||
(self.pids_limit, 16, 4_096, "PID"),
|
||||
(self.shm_bytes, 64 * 1024**2, 128 * 1024**3, "shared-memory"),
|
||||
(self.tmpfs_bytes, 64 * 1024**2, 64 * 1024**3, "tmpfs"),
|
||||
):
|
||||
if isinstance(value, bool) or not minimum <= value <= maximum:
|
||||
raise InstalledLabPackageError(f"package container {label} limit is invalid")
|
||||
if isinstance(self.timeout_seconds, bool) or not 1 <= self.timeout_seconds <= 24 * 60 * 60:
|
||||
raise InstalledLabPackageError("package container timeout is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"container_id": self.container_id,
|
||||
"role": self.role,
|
||||
"image_sha256": self.image_sha256,
|
||||
"argv": list(self.argv),
|
||||
"depends_on": list(self.depends_on),
|
||||
"mounts": [mount.as_dict() for mount in self.mounts],
|
||||
"network": self.network,
|
||||
"gpu_count": self.gpu_count,
|
||||
"memory_bytes": self.memory_bytes,
|
||||
"nano_cpus": self.nano_cpus,
|
||||
"pids_limit": self.pids_limit,
|
||||
"shm_bytes": self.shm_bytes,
|
||||
"tmpfs_bytes": self.tmpfs_bytes,
|
||||
"timeout_seconds": self.timeout_seconds,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabPackage:
|
||||
package_id: str
|
||||
package_version: int
|
||||
package_sha256: str
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
runtime_candidate_sha256: str
|
||||
source_adapter_sha256: str
|
||||
result_contract_sha256: str
|
||||
executor_identity: RecordedExecutorIdentity
|
||||
execution_mode: InstalledLabExecutionMode
|
||||
asset_ids: tuple[str, ...]
|
||||
containers: tuple[InstalledLabContainer, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value, label in (
|
||||
(self.package_id, "package id"),
|
||||
(self.setup_id, "package setup id"),
|
||||
(self.definition_id, "package definition id"),
|
||||
):
|
||||
_pattern(value, _IDENTIFIER, label)
|
||||
for value, label in (
|
||||
(self.package_sha256, "package sha256"),
|
||||
(self.definition_sha256, "package definition sha256"),
|
||||
(self.runtime_candidate_sha256, "runtime candidate sha256"),
|
||||
(self.source_adapter_sha256, "source adapter sha256"),
|
||||
(self.result_contract_sha256, "result contract sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
for numeric_value, label in (
|
||||
(self.package_version, "package version"),
|
||||
(self.definition_version, "package definition version"),
|
||||
):
|
||||
if (
|
||||
isinstance(numeric_value, bool)
|
||||
or not isinstance(numeric_value, int)
|
||||
or numeric_value < 1
|
||||
):
|
||||
raise InstalledLabPackageError(f"{label} is invalid")
|
||||
if self.execution_mode not in ("single-container", "fixed-stack"):
|
||||
raise InstalledLabPackageError("package execution mode is invalid")
|
||||
if self.asset_ids != tuple(sorted(self.asset_ids)) or len(self.asset_ids) != len(
|
||||
set(self.asset_ids)
|
||||
):
|
||||
raise InstalledLabPackageError("package asset ids must be canonical")
|
||||
for asset_id in self.asset_ids:
|
||||
_pattern(asset_id, _ASSET_ID, "package asset id")
|
||||
container_ids = tuple(container.container_id for container in self.containers)
|
||||
if (
|
||||
not container_ids
|
||||
or container_ids != tuple(sorted(container_ids))
|
||||
or len(container_ids) != len(set(container_ids))
|
||||
):
|
||||
raise InstalledLabPackageError("package containers must be canonical and unique")
|
||||
writers = tuple(
|
||||
container for container in self.containers if container.role == "result-writer"
|
||||
)
|
||||
if len(writers) != 1:
|
||||
raise InstalledLabPackageError("package requires exactly one result writer")
|
||||
if self.execution_mode == "single-container" and (
|
||||
len(self.containers) != 1 or self.containers[0].role != "result-writer"
|
||||
):
|
||||
raise InstalledLabPackageError(
|
||||
"single-container package must contain one result writer"
|
||||
)
|
||||
if self.execution_mode == "fixed-stack" and len(self.containers) < 2:
|
||||
raise InstalledLabPackageError("fixed-stack package requires multiple containers")
|
||||
self._verify_topology()
|
||||
mounted_assets = {
|
||||
mount.asset_id for container in self.containers for mount in container.mounts
|
||||
}
|
||||
if not mounted_assets.issubset(set(self.asset_ids)):
|
||||
raise InstalledLabPackageError("package mounts reference undeclared assets")
|
||||
if self.package_sha256 != canonical_sha256(self.identity_document()):
|
||||
raise InstalledLabPackageError("installed LAB package digest changed")
|
||||
|
||||
def _verify_topology(self) -> None:
|
||||
by_id = {container.container_id: container for container in self.containers}
|
||||
for container in self.containers:
|
||||
if container.container_id in container.depends_on or any(
|
||||
dependency not in by_id for dependency in container.depends_on
|
||||
):
|
||||
raise InstalledLabPackageError("package dependency is invalid")
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def visit(container_id: str) -> None:
|
||||
if container_id in visiting:
|
||||
raise InstalledLabPackageError("package container graph contains a cycle")
|
||||
if container_id in visited:
|
||||
return
|
||||
visiting.add(container_id)
|
||||
for dependency in by_id[container_id].depends_on:
|
||||
visit(dependency)
|
||||
visiting.remove(container_id)
|
||||
visited.add(container_id)
|
||||
|
||||
for container_id in by_id:
|
||||
visit(container_id)
|
||||
writer = next(
|
||||
container for container in self.containers if container.role == "result-writer"
|
||||
)
|
||||
writer_ancestors: set[str] = set()
|
||||
|
||||
def collect(container_id: str) -> None:
|
||||
for dependency in by_id[container_id].depends_on:
|
||||
if dependency not in writer_ancestors:
|
||||
writer_ancestors.add(dependency)
|
||||
collect(dependency)
|
||||
|
||||
collect(writer.container_id)
|
||||
steps = {
|
||||
container.container_id for container in self.containers if container.role == "step"
|
||||
}
|
||||
if writer_ancestors != steps:
|
||||
raise InstalledLabPackageError(
|
||||
"package result writer must depend on every execution step"
|
||||
)
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return _package_identity_document(
|
||||
package_id=self.package_id,
|
||||
package_version=self.package_version,
|
||||
setup_id=self.setup_id,
|
||||
definition_id=self.definition_id,
|
||||
definition_version=self.definition_version,
|
||||
definition_sha256=self.definition_sha256,
|
||||
runtime_candidate_sha256=self.runtime_candidate_sha256,
|
||||
source_adapter_sha256=self.source_adapter_sha256,
|
||||
result_contract_sha256=self.result_contract_sha256,
|
||||
executor_identity=self.executor_identity,
|
||||
execution_mode=self.execution_mode,
|
||||
asset_ids=self.asset_ids,
|
||||
containers=self.containers,
|
||||
)
|
||||
|
||||
def bind(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
candidate: PortableWorkerRuntimeCandidate,
|
||||
) -> None:
|
||||
candidate.bind_definition(definition)
|
||||
if not candidate.ready:
|
||||
raise InstalledLabPackageError("installed package binds a blocked runtime")
|
||||
if (
|
||||
self.setup_id != definition.setup_id
|
||||
or self.definition_id != definition.definition_id
|
||||
or self.definition_version != definition.version
|
||||
or self.definition_sha256 != definition.definition_sha256
|
||||
or self.runtime_candidate_sha256 != candidate.candidate_sha256
|
||||
or self.source_adapter_sha256 != definition.source_adapter.contract_sha256
|
||||
or self.result_contract_sha256 != definition.result_contract.contract_sha256
|
||||
or self.executor_identity != candidate.executor_identity()
|
||||
):
|
||||
raise InstalledLabPackageError(
|
||||
"installed package differs from its definition or runtime candidate"
|
||||
)
|
||||
candidate_assets = {asset.asset_id: asset for asset in candidate.reusable_assets}
|
||||
if set(self.asset_ids) != set(candidate_assets):
|
||||
raise InstalledLabPackageError(
|
||||
"installed package asset inventory differs from its runtime candidate"
|
||||
)
|
||||
admitted_images = {
|
||||
asset.sha256 for asset in candidate.reusable_assets if asset.kind == "container-image"
|
||||
}
|
||||
if candidate.executor is not None:
|
||||
admitted_images.add(candidate.executor.image_sha256)
|
||||
if any(container.image_sha256 not in admitted_images for container in self.containers):
|
||||
raise InstalledLabPackageError("package container image is not runtime-admitted")
|
||||
for container in self.containers:
|
||||
for mount in container.mounts:
|
||||
if candidate_assets[mount.asset_id].kind == "container-image":
|
||||
raise InstalledLabPackageError("container images cannot be mounted as files")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabPackageRegistry:
|
||||
packages: tuple[InstalledLabPackage, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.packages:
|
||||
raise InstalledLabPackageError("installed LAB package registry is empty")
|
||||
for values, label in (
|
||||
([package.package_id for package in self.packages], "package ids"),
|
||||
([package.package_sha256 for package in self.packages], "package digests"),
|
||||
(
|
||||
[(package.setup_id, package.definition_sha256) for package in self.packages],
|
||||
"package definition bindings",
|
||||
),
|
||||
):
|
||||
if len(values) != len(set(values)):
|
||||
raise InstalledLabPackageError(f"installed LAB {label} must be unique")
|
||||
|
||||
@classmethod
|
||||
def from_file(
|
||||
cls,
|
||||
path: Path,
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
runtime_registry: PortableWorkerRuntimeRegistry,
|
||||
) -> InstalledLabPackageRegistry:
|
||||
candidate_path = path.expanduser().absolute()
|
||||
try:
|
||||
if candidate_path.is_symlink() or not candidate_path.is_file():
|
||||
raise InstalledLabPackageError(
|
||||
"installed LAB package registry must be a regular file"
|
||||
)
|
||||
payload = candidate_path.read_bytes()
|
||||
if not 0 < len(payload) <= _MAX_REGISTRY_BYTES:
|
||||
raise InstalledLabPackageError("installed LAB package registry size is invalid")
|
||||
document: object = json.loads(payload.decode("utf-8"))
|
||||
except InstalledLabPackageError:
|
||||
raise
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise InstalledLabPackageError("installed LAB package registry is unreadable") from exc
|
||||
_reject_forbidden_keys(document)
|
||||
root = _object(document, "installed LAB package registry")
|
||||
_exact_keys(root, {"schema_version", "packages"}, "installed LAB package registry")
|
||||
if root["schema_version"] != INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA:
|
||||
raise InstalledLabPackageError("installed LAB package registry schema is invalid")
|
||||
registry = cls(tuple(_package(value) for value in _array(root["packages"], "packages")))
|
||||
for package in registry.packages:
|
||||
definition = definitions.resolve(package.setup_id, package.definition_sha256)
|
||||
candidate = runtime_registry.resolve(
|
||||
package.setup_id,
|
||||
package.definition_sha256,
|
||||
)
|
||||
package.bind(definition, candidate)
|
||||
return registry
|
||||
|
||||
def resolve(self, setup_id: str, definition_sha256: str) -> InstalledLabPackage:
|
||||
for package in self.packages:
|
||||
if package.setup_id == setup_id and package.definition_sha256 == definition_sha256:
|
||||
return package
|
||||
raise InstalledLabPackageError("installed LAB package is unavailable")
|
||||
|
||||
|
||||
def seal_installed_lab_package(
|
||||
*,
|
||||
package_id: str,
|
||||
package_version: int,
|
||||
setup_id: str,
|
||||
definition_id: str,
|
||||
definition_version: int,
|
||||
definition_sha256: str,
|
||||
runtime_candidate_sha256: str,
|
||||
source_adapter_sha256: str,
|
||||
result_contract_sha256: str,
|
||||
executor_identity: RecordedExecutorIdentity,
|
||||
execution_mode: InstalledLabExecutionMode,
|
||||
asset_ids: tuple[str, ...],
|
||||
containers: tuple[InstalledLabContainer, ...],
|
||||
) -> InstalledLabPackage:
|
||||
canonical_asset_ids = tuple(sorted(asset_ids))
|
||||
canonical_containers = tuple(sorted(containers, key=lambda value: value.container_id))
|
||||
package_sha256 = canonical_sha256(
|
||||
_package_identity_document(
|
||||
package_id=package_id,
|
||||
package_version=package_version,
|
||||
setup_id=setup_id,
|
||||
definition_id=definition_id,
|
||||
definition_version=definition_version,
|
||||
definition_sha256=definition_sha256,
|
||||
runtime_candidate_sha256=runtime_candidate_sha256,
|
||||
source_adapter_sha256=source_adapter_sha256,
|
||||
result_contract_sha256=result_contract_sha256,
|
||||
executor_identity=executor_identity,
|
||||
execution_mode=execution_mode,
|
||||
asset_ids=canonical_asset_ids,
|
||||
containers=canonical_containers,
|
||||
)
|
||||
)
|
||||
return InstalledLabPackage(
|
||||
package_id=package_id,
|
||||
package_version=package_version,
|
||||
package_sha256=package_sha256,
|
||||
setup_id=setup_id,
|
||||
definition_id=definition_id,
|
||||
definition_version=definition_version,
|
||||
definition_sha256=definition_sha256,
|
||||
runtime_candidate_sha256=runtime_candidate_sha256,
|
||||
source_adapter_sha256=source_adapter_sha256,
|
||||
result_contract_sha256=result_contract_sha256,
|
||||
executor_identity=executor_identity,
|
||||
execution_mode=execution_mode,
|
||||
asset_ids=canonical_asset_ids,
|
||||
containers=canonical_containers,
|
||||
)
|
||||
|
||||
|
||||
def _package_identity_document(
|
||||
*,
|
||||
package_id: str,
|
||||
package_version: int,
|
||||
setup_id: str,
|
||||
definition_id: str,
|
||||
definition_version: int,
|
||||
definition_sha256: str,
|
||||
runtime_candidate_sha256: str,
|
||||
source_adapter_sha256: str,
|
||||
result_contract_sha256: str,
|
||||
executor_identity: RecordedExecutorIdentity,
|
||||
execution_mode: InstalledLabExecutionMode,
|
||||
asset_ids: tuple[str, ...],
|
||||
containers: tuple[InstalledLabContainer, ...],
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": INSTALLED_LAB_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"package_version": package_version,
|
||||
"setup_id": setup_id,
|
||||
"definition_id": definition_id,
|
||||
"definition_version": definition_version,
|
||||
"definition_sha256": definition_sha256,
|
||||
"runtime_candidate_sha256": runtime_candidate_sha256,
|
||||
"source_adapter_sha256": source_adapter_sha256,
|
||||
"result_contract_sha256": result_contract_sha256,
|
||||
"executor_identity": executor_identity.as_dict(),
|
||||
"container_io": {
|
||||
"schema_version": INSTALLED_LAB_CONTAINER_IO_SCHEMA,
|
||||
"source_root": INSTALLED_LAB_SOURCE_ROOT,
|
||||
"plan_path": INSTALLED_LAB_PLAN_PATH,
|
||||
"step_input_root": INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
"result_root": INSTALLED_LAB_RESULT_ROOT,
|
||||
"work_root": INSTALLED_LAB_WORK_ROOT,
|
||||
},
|
||||
"execution_mode": execution_mode,
|
||||
"asset_ids": list(asset_ids),
|
||||
"containers": [container.as_dict() for container in containers],
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
def _package(value: object) -> InstalledLabPackage:
|
||||
row = _object(value, "installed LAB package")
|
||||
_exact_keys(
|
||||
row,
|
||||
{
|
||||
"schema_version",
|
||||
"package_id",
|
||||
"package_version",
|
||||
"package_sha256",
|
||||
"setup_id",
|
||||
"definition_id",
|
||||
"definition_version",
|
||||
"definition_sha256",
|
||||
"runtime_candidate_sha256",
|
||||
"source_adapter_sha256",
|
||||
"result_contract_sha256",
|
||||
"executor_identity",
|
||||
"container_io",
|
||||
"execution_mode",
|
||||
"asset_ids",
|
||||
"containers",
|
||||
"authority",
|
||||
},
|
||||
"installed LAB package",
|
||||
)
|
||||
if row["schema_version"] != INSTALLED_LAB_PACKAGE_SCHEMA:
|
||||
raise InstalledLabPackageError("installed LAB package schema is invalid")
|
||||
if row["authority"] != _AUTHORITY:
|
||||
raise InstalledLabPackageError("installed LAB package authority changed")
|
||||
if row["container_io"] != {
|
||||
"schema_version": INSTALLED_LAB_CONTAINER_IO_SCHEMA,
|
||||
"source_root": INSTALLED_LAB_SOURCE_ROOT,
|
||||
"plan_path": INSTALLED_LAB_PLAN_PATH,
|
||||
"step_input_root": INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
"result_root": INSTALLED_LAB_RESULT_ROOT,
|
||||
"work_root": INSTALLED_LAB_WORK_ROOT,
|
||||
}:
|
||||
raise InstalledLabPackageError("installed LAB container I/O contract changed")
|
||||
executor = _object(row["executor_identity"], "executor identity")
|
||||
_exact_keys(
|
||||
executor,
|
||||
{
|
||||
"release_sha256",
|
||||
"image_sha256",
|
||||
"model_manifest_sha256",
|
||||
"resource_profile_sha256",
|
||||
},
|
||||
"executor identity",
|
||||
)
|
||||
mode = row["execution_mode"]
|
||||
if mode not in ("single-container", "fixed-stack"):
|
||||
raise InstalledLabPackageError("installed LAB execution mode is invalid")
|
||||
return InstalledLabPackage(
|
||||
package_id=_string(row["package_id"], "package id"),
|
||||
package_version=_integer(row["package_version"], "package version"),
|
||||
package_sha256=_string(row["package_sha256"], "package sha256"),
|
||||
setup_id=_string(row["setup_id"], "setup id"),
|
||||
definition_id=_string(row["definition_id"], "definition id"),
|
||||
definition_version=_integer(row["definition_version"], "definition version"),
|
||||
definition_sha256=_string(row["definition_sha256"], "definition sha256"),
|
||||
runtime_candidate_sha256=_string(
|
||||
row["runtime_candidate_sha256"], "runtime candidate sha256"
|
||||
),
|
||||
source_adapter_sha256=_string(row["source_adapter_sha256"], "source adapter sha256"),
|
||||
result_contract_sha256=_string(row["result_contract_sha256"], "result contract sha256"),
|
||||
executor_identity=RecordedExecutorIdentity(
|
||||
release_sha256=_string(executor["release_sha256"], "executor release sha256"),
|
||||
image_sha256=_string(executor["image_sha256"], "executor image sha256"),
|
||||
model_manifest_sha256=_string(
|
||||
executor["model_manifest_sha256"], "model manifest sha256"
|
||||
),
|
||||
resource_profile_sha256=_string(
|
||||
executor["resource_profile_sha256"], "resource profile sha256"
|
||||
),
|
||||
),
|
||||
execution_mode=mode,
|
||||
asset_ids=tuple(
|
||||
_string(item, "package asset id")
|
||||
for item in _array(row["asset_ids"], "package asset ids")
|
||||
),
|
||||
containers=tuple(
|
||||
_container(item) for item in _array(row["containers"], "package containers")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _container(value: object) -> InstalledLabContainer:
|
||||
row = _object(value, "package container")
|
||||
_exact_keys(
|
||||
row,
|
||||
{
|
||||
"container_id",
|
||||
"role",
|
||||
"image_sha256",
|
||||
"argv",
|
||||
"depends_on",
|
||||
"mounts",
|
||||
"network",
|
||||
"gpu_count",
|
||||
"memory_bytes",
|
||||
"nano_cpus",
|
||||
"pids_limit",
|
||||
"shm_bytes",
|
||||
"tmpfs_bytes",
|
||||
"timeout_seconds",
|
||||
},
|
||||
"package container",
|
||||
)
|
||||
role = row["role"]
|
||||
network = row["network"]
|
||||
if role not in ("step", "result-writer") or network != "none":
|
||||
raise InstalledLabPackageError("package container enum is invalid")
|
||||
return InstalledLabContainer(
|
||||
container_id=_string(row["container_id"], "package container id"),
|
||||
role=role,
|
||||
image_sha256=_string(row["image_sha256"], "package image sha256"),
|
||||
argv=tuple(
|
||||
_string(item, "package argument") for item in _array(row["argv"], "package argv")
|
||||
),
|
||||
depends_on=tuple(
|
||||
_string(item, "package dependency")
|
||||
for item in _array(row["depends_on"], "package dependencies")
|
||||
),
|
||||
mounts=tuple(_mount(item) for item in _array(row["mounts"], "package mounts")),
|
||||
network=network,
|
||||
gpu_count=_integer(row["gpu_count"], "package GPU count"),
|
||||
memory_bytes=_integer(row["memory_bytes"], "package memory limit"),
|
||||
nano_cpus=_integer(row["nano_cpus"], "package CPU limit"),
|
||||
pids_limit=_integer(row["pids_limit"], "package PID limit"),
|
||||
shm_bytes=_integer(row["shm_bytes"], "package shared-memory limit"),
|
||||
tmpfs_bytes=_integer(row["tmpfs_bytes"], "package tmpfs limit"),
|
||||
timeout_seconds=_integer(row["timeout_seconds"], "package timeout"),
|
||||
)
|
||||
|
||||
|
||||
def _mount(value: object) -> InstalledLabPackageMount:
|
||||
row = _object(value, "package mount")
|
||||
_exact_keys(row, {"asset_id", "target", "read_only"}, "package mount")
|
||||
if row["read_only"] is not True:
|
||||
raise InstalledLabPackageError("package mount must be read-only")
|
||||
return InstalledLabPackageMount(
|
||||
asset_id=_string(row["asset_id"], "package mount asset id"),
|
||||
target=_string(row["target"], "package mount target"),
|
||||
)
|
||||
|
||||
|
||||
def _reject_forbidden_keys(value: object) -> None:
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
if not isinstance(key, str) or key.lower() in _FORBIDDEN_KEYS:
|
||||
raise InstalledLabPackageError(
|
||||
"installed LAB package contains a forbidden host or secret input"
|
||||
)
|
||||
_reject_forbidden_keys(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
_reject_forbidden_keys(child)
|
||||
|
||||
|
||||
def _pattern(value: str, pattern: re.Pattern[str], label: str) -> None:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise InstalledLabPackageError(f"{label} is invalid")
|
||||
|
||||
|
||||
def _digest(value: str, label: str) -> None:
|
||||
_pattern(value, _SHA256, label)
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||
raise InstalledLabPackageError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _array(value: object, label: str) -> list[object]:
|
||||
if not isinstance(value, list):
|
||||
raise InstalledLabPackageError(f"{label} must be an array")
|
||||
return value
|
||||
|
||||
|
||||
def _string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise InstalledLabPackageError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _integer(value: object, label: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise InstalledLabPackageError(f"{label} must be an integer")
|
||||
return value
|
||||
|
||||
|
||||
def _exact_keys(row: dict[str, object], expected: set[str], label: str) -> None:
|
||||
if set(row) != expected:
|
||||
raise InstalledLabPackageError(f"{label} keys are invalid")
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Container entrypoint for the generic installed-LAB Worker agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from k1link.observatory import installed_lab_worker_service
|
||||
from k1link.observatory.worker_container_proxy import (
|
||||
FixedObservatoryContainerLoopbackProxy,
|
||||
)
|
||||
|
||||
|
||||
def main(arguments: Sequence[str] | None = None) -> int:
|
||||
"""Run the fixed loopback bridge around the package-driven service."""
|
||||
|
||||
with FixedObservatoryContainerLoopbackProxy():
|
||||
return installed_lab_worker_service.main(arguments)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,466 @@
|
||||
"""Generic Worker entrypoint for independently installed LAB packages.
|
||||
|
||||
The environment selects only immutable registry files, one reviewed local
|
||||
asset-binding file, the Worker work roots, and the existing transport settings.
|
||||
Executable images, argv, topology, and limits live in the content-addressed
|
||||
package registry and cannot arrive through a queued job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import stat
|
||||
import sys
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from types import FrameType
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.installed_lab_package_runner import (
|
||||
DockerEngineInstalledLabLauncher,
|
||||
InstalledLabLocalAssetBinding,
|
||||
InstalledLabPackageExecutorFactory,
|
||||
)
|
||||
from k1link.observatory.installed_lab_packages import InstalledLabPackageRegistry
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import (
|
||||
PortableWorkerResultDraft,
|
||||
PortableWorkerRuntimeRegistry,
|
||||
PortableWorkerSourceStage,
|
||||
)
|
||||
from k1link.observatory.worker_agent import (
|
||||
ObservatoryWorkerExecutionResult,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
from k1link.observatory.worker_service import (
|
||||
OBSERVATORY_WORKER_WORK_ROOT_ENV,
|
||||
InstalledObservatoryWorkerService,
|
||||
ObservatoryWorkerServiceConfiguration,
|
||||
build_ready_executor_registry_from_packages,
|
||||
compose_installed_observatory_worker_service_from_packages,
|
||||
)
|
||||
|
||||
INSTALLED_LAB_ASSET_BINDINGS_SCHEMA: Final = (
|
||||
"missioncore.observatory-installed-lab-asset-bindings/v1"
|
||||
)
|
||||
INSTALLED_LAB_VALIDATION_RECEIPT_SCHEMA: Final = (
|
||||
"missioncore.observatory-installed-lab-worker-validation/v1"
|
||||
)
|
||||
INSTALLED_LAB_DEFINITIONS_FILE_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE"
|
||||
)
|
||||
INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE"
|
||||
)
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE"
|
||||
)
|
||||
INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE"
|
||||
)
|
||||
_MAX_BINDINGS_BYTES: Final = 1024 * 1024
|
||||
|
||||
|
||||
class InstalledLabWorkerCompositionError(RuntimeError):
|
||||
"""The generic installed-LAB Worker configuration is not exact."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabWorkerAssetBindings:
|
||||
engine_work_root: str
|
||||
assets: tuple[InstalledLabLocalAssetBinding, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
asset_ids = tuple(asset.asset_id for asset in self.assets)
|
||||
if asset_ids != tuple(sorted(asset_ids)) or len(asset_ids) != len(set(asset_ids)):
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
"installed LAB asset bindings must be canonical and unique"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> InstalledLabWorkerAssetBindings:
|
||||
document = _read_object(path, "installed LAB asset bindings")
|
||||
_exact_keys(
|
||||
document,
|
||||
{"schema_version", "engine_work_root", "assets"},
|
||||
"installed LAB asset bindings",
|
||||
)
|
||||
if document["schema_version"] != INSTALLED_LAB_ASSET_BINDINGS_SCHEMA:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
"installed LAB asset binding schema changed"
|
||||
)
|
||||
values = document["assets"]
|
||||
if not isinstance(values, list) or not values:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
"installed LAB asset binding inventory is empty"
|
||||
)
|
||||
assets = tuple(
|
||||
sorted((_asset_binding(value) for value in values), key=lambda item: item.asset_id)
|
||||
)
|
||||
return cls(
|
||||
engine_work_root=_text(document["engine_work_root"], "engine work root"),
|
||||
assets=assets,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabWorkerEntrypointConfiguration:
|
||||
worker: ObservatoryWorkerServiceConfiguration
|
||||
definitions_file: Path
|
||||
runtime_registry_file: Path
|
||||
package_registry_file: Path
|
||||
asset_bindings_file: Path
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> InstalledLabWorkerEntrypointConfiguration:
|
||||
values = os.environ if environment is None else environment
|
||||
return cls(
|
||||
worker=ObservatoryWorkerServiceConfiguration.from_environment(values),
|
||||
definitions_file=_required_path(values, INSTALLED_LAB_DEFINITIONS_FILE_ENV),
|
||||
runtime_registry_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV,
|
||||
),
|
||||
package_registry_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV,
|
||||
),
|
||||
asset_bindings_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstalledLabWorkerValidationConfiguration:
|
||||
work_root: Path
|
||||
definitions_file: Path
|
||||
runtime_registry_file: Path
|
||||
package_registry_file: Path
|
||||
asset_bindings_file: Path
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> InstalledLabWorkerValidationConfiguration:
|
||||
values = os.environ if environment is None else environment
|
||||
return cls(
|
||||
work_root=_required_path(values, OBSERVATORY_WORKER_WORK_ROOT_ENV),
|
||||
definitions_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_DEFINITIONS_FILE_ENV,
|
||||
),
|
||||
runtime_registry_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_RUNTIME_REGISTRY_FILE_ENV,
|
||||
),
|
||||
package_registry_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_PACKAGE_REGISTRY_FILE_ENV,
|
||||
),
|
||||
asset_bindings_file=_required_path(
|
||||
values,
|
||||
INSTALLED_LAB_ASSET_BINDINGS_FILE_ENV,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def compose_installed_lab_worker_service(
|
||||
configuration: InstalledLabWorkerEntrypointConfiguration,
|
||||
) -> InstalledObservatoryWorkerService:
|
||||
definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file)
|
||||
runtime = PortableWorkerRuntimeRegistry.from_file(
|
||||
configuration.runtime_registry_file,
|
||||
definitions=definitions,
|
||||
)
|
||||
packages = InstalledLabPackageRegistry.from_file(
|
||||
configuration.package_registry_file,
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
)
|
||||
bindings = InstalledLabWorkerAssetBindings.from_file(configuration.asset_bindings_file)
|
||||
return compose_installed_observatory_worker_service_from_packages(
|
||||
configuration=configuration.worker,
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
packages=packages,
|
||||
executor_factory=InstalledLabPackageExecutorFactory(
|
||||
local_assets=bindings.assets,
|
||||
engine_work_root=bindings.engine_work_root,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def validate_installed_lab_worker(
|
||||
configuration: InstalledLabWorkerValidationConfiguration,
|
||||
*,
|
||||
launcher: DockerEngineInstalledLabLauncher | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Validate one installed package set without credentials or queue access."""
|
||||
|
||||
work_root = _real_directory(configuration.work_root, "installed LAB work root")
|
||||
definitions = PortableRunDefinitionRegistry.from_file(configuration.definitions_file)
|
||||
runtime = PortableWorkerRuntimeRegistry.from_file(
|
||||
configuration.runtime_registry_file,
|
||||
definitions=definitions,
|
||||
)
|
||||
packages = InstalledLabPackageRegistry.from_file(
|
||||
configuration.package_registry_file,
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
)
|
||||
bindings = InstalledLabWorkerAssetBindings.from_file(configuration.asset_bindings_file)
|
||||
image_sha256s = tuple(
|
||||
sorted(
|
||||
{
|
||||
container.image_sha256
|
||||
for package in packages.packages
|
||||
for container in package.containers
|
||||
}
|
||||
)
|
||||
)
|
||||
image_launcher = DockerEngineInstalledLabLauncher() if launcher is None else launcher
|
||||
image_launcher.verify_images(image_sha256s)
|
||||
offline = _OfflineWorkerBoundary()
|
||||
executors = build_ready_executor_registry_from_packages(
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime,
|
||||
packages=packages,
|
||||
executor_factory=InstalledLabPackageExecutorFactory(
|
||||
local_assets=bindings.assets,
|
||||
engine_work_root=bindings.engine_work_root,
|
||||
launcher=image_launcher,
|
||||
),
|
||||
source_transport=offline,
|
||||
result_transport=offline,
|
||||
work_root=work_root,
|
||||
)
|
||||
expected_identities = tuple(
|
||||
sorted(package.executor_identity for package in packages.packages)
|
||||
)
|
||||
if executors.supported_identities != expected_identities:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
"installed LAB validation capability inventory changed"
|
||||
)
|
||||
package_receipts: list[dict[str, object]] = []
|
||||
for package in packages.packages:
|
||||
candidate = runtime.resolve(package.setup_id, package.definition_sha256)
|
||||
package_receipts.append(
|
||||
{
|
||||
"package_id": package.package_id,
|
||||
"package_version": package.package_version,
|
||||
"package_sha256": package.package_sha256,
|
||||
"setup_id": package.setup_id,
|
||||
"definition_sha256": package.definition_sha256,
|
||||
"runtime_candidate_sha256": package.runtime_candidate_sha256,
|
||||
"executor": package.executor_identity.as_dict(),
|
||||
"assets": [
|
||||
{
|
||||
"asset_id": asset.asset_id,
|
||||
"kind": asset.kind,
|
||||
"sha256": asset.sha256,
|
||||
}
|
||||
for asset in candidate.reusable_assets
|
||||
],
|
||||
"containers": [
|
||||
{
|
||||
"container_id": container.container_id,
|
||||
"role": container.role,
|
||||
"image_sha256": container.image_sha256,
|
||||
}
|
||||
for container in package.containers
|
||||
],
|
||||
}
|
||||
)
|
||||
identity = {
|
||||
"schema_version": INSTALLED_LAB_VALIDATION_RECEIPT_SCHEMA,
|
||||
"state": "ready",
|
||||
"packages": package_receipts,
|
||||
"verified_image_sha256s": list(image_sha256s),
|
||||
"supported_executor_identities": [
|
||||
identity.as_dict() for identity in executors.supported_identities
|
||||
],
|
||||
"checks": {
|
||||
"registry_binding": "passed",
|
||||
"local_asset_identity": "passed",
|
||||
"docker_image_identity": "passed",
|
||||
"backend_contacted": False,
|
||||
"claim_attempted": False,
|
||||
},
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
return {**identity, "receipt_sha256": canonical_sha256(identity)}
|
||||
|
||||
|
||||
def run_installed_lab_worker(
|
||||
service: InstalledObservatoryWorkerService,
|
||||
*,
|
||||
stop: Event,
|
||||
once: bool = False,
|
||||
) -> None:
|
||||
if once:
|
||||
try:
|
||||
service.agent.run_once()
|
||||
finally:
|
||||
service.close()
|
||||
return
|
||||
service.run(stop=stop)
|
||||
|
||||
|
||||
def main(arguments: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--once", action="store_true", help="Run one claim cycle and exit.")
|
||||
mode.add_argument(
|
||||
"--validate-only",
|
||||
action="store_true",
|
||||
help="Validate local packages and print a receipt without contacting Mission Core.",
|
||||
)
|
||||
options = parser.parse_args(arguments)
|
||||
if cast(bool, options.validate_only):
|
||||
receipt = validate_installed_lab_worker(
|
||||
InstalledLabWorkerValidationConfiguration.from_environment()
|
||||
)
|
||||
sys.stdout.buffer.write(canonical_json(receipt) + b"\n")
|
||||
return 0
|
||||
service = compose_installed_lab_worker_service(
|
||||
InstalledLabWorkerEntrypointConfiguration.from_environment()
|
||||
)
|
||||
stop = Event()
|
||||
with _posix_shutdown_signals(stop):
|
||||
run_installed_lab_worker(service, stop=stop, once=cast(bool, options.once))
|
||||
return 0
|
||||
|
||||
|
||||
def _asset_binding(value: object) -> InstalledLabLocalAssetBinding:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise InstalledLabWorkerCompositionError("installed LAB asset binding is not an object")
|
||||
_exact_keys(
|
||||
value,
|
||||
{"asset_id", "controller_path", "engine_path", "image_sha256"},
|
||||
"installed LAB asset binding",
|
||||
)
|
||||
controller_value = value["controller_path"]
|
||||
engine_value = value["engine_path"]
|
||||
image_value = value["image_sha256"]
|
||||
try:
|
||||
if controller_value is None and engine_value is None and isinstance(image_value, str):
|
||||
return InstalledLabLocalAssetBinding(
|
||||
asset_id=_text(value["asset_id"], "installed LAB asset id"),
|
||||
image_sha256=image_value,
|
||||
)
|
||||
if (
|
||||
isinstance(controller_value, str)
|
||||
and isinstance(engine_value, str)
|
||||
and image_value is None
|
||||
):
|
||||
return InstalledLabLocalAssetBinding(
|
||||
asset_id=_text(value["asset_id"], "installed LAB asset id"),
|
||||
controller_path=Path(controller_value),
|
||||
engine_path=engine_value,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
"installed LAB asset locator is invalid"
|
||||
) from exc
|
||||
raise InstalledLabWorkerCompositionError("installed LAB asset locator is ambiguous")
|
||||
|
||||
|
||||
def _read_object(path: Path, label: str) -> dict[str, object]:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
if candidate.is_symlink() or not candidate.is_file():
|
||||
raise InstalledLabWorkerCompositionError(f"{label} must be a regular file")
|
||||
payload = candidate.read_bytes()
|
||||
if not 0 < len(payload) <= _MAX_BINDINGS_BYTES:
|
||||
raise InstalledLabWorkerCompositionError(f"{label} size is invalid")
|
||||
value: object = json.loads(payload.decode("utf-8"))
|
||||
except InstalledLabWorkerCompositionError:
|
||||
raise
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise InstalledLabWorkerCompositionError(f"{label} is unreadable") from exc
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise InstalledLabWorkerCompositionError(f"{label} is not an object")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _required_path(values: Mapping[str, str], name: str) -> Path:
|
||||
value = values.get(name)
|
||||
if value is None or not value or not Path(value).is_absolute():
|
||||
raise InstalledLabWorkerCompositionError(f"{name} must name an absolute path")
|
||||
return Path(value)
|
||||
|
||||
|
||||
def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||
if set(value) != expected:
|
||||
raise InstalledLabWorkerCompositionError(f"{label} fields changed")
|
||||
|
||||
|
||||
def _text(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value or value != value.strip():
|
||||
raise InstalledLabWorkerCompositionError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
try:
|
||||
metadata = candidate.lstat()
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise InstalledLabWorkerCompositionError(f"{label} is unavailable") from exc
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise InstalledLabWorkerCompositionError(f"{label} is not a real directory")
|
||||
return resolved
|
||||
|
||||
|
||||
class _OfflineWorkerBoundary:
|
||||
def materialize(self, job: SealedObservatoryRecordedJob) -> PortableWorkerSourceStage:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
f"offline validation cannot materialize job {job.job_id}"
|
||||
)
|
||||
|
||||
def publish(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
draft: PortableWorkerResultDraft,
|
||||
) -> ObservatoryWorkerExecutionResult:
|
||||
raise InstalledLabWorkerCompositionError(
|
||||
f"offline validation cannot publish job {job.job_id} from {draft.root}"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _posix_shutdown_signals(stop: Event) -> Iterator[None]:
|
||||
def request_stop(_signal: int, _frame: FrameType | None) -> None:
|
||||
stop.set()
|
||||
|
||||
previous_int = signal.signal(signal.SIGINT, request_stop)
|
||||
previous_term = signal.signal(signal.SIGTERM, request_stop)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
signal.signal(signal.SIGINT, previous_int)
|
||||
signal.signal(signal.SIGTERM, previous_term)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,499 @@
|
||||
"""Package-owned prepare/assemble steps for the generic LAB V1 stack.
|
||||
|
||||
The generic Worker only supplies the sealed source, run plan, dependency
|
||||
outputs, and reviewed assets. This module is the LAB package's own adapter: it
|
||||
turns those standard inputs into the existing EoMT/DDRNet component contracts
|
||||
and finally emits the common portable result package.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
INSTALLED_LAB_PLAN_PATH,
|
||||
INSTALLED_LAB_RESULT_ROOT,
|
||||
INSTALLED_LAB_SOURCE_ROOT,
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_executor import (
|
||||
PortableLabV1OrchestrationPlan,
|
||||
assemble_lab_v1_result_v2,
|
||||
package_lab_v1_result,
|
||||
portable_lab_v1_orchestration_plan_from_document,
|
||||
portable_lab_v1_source_input_from_document,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_local_runners import (
|
||||
PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_worker import (
|
||||
_SealedPackageJobView,
|
||||
materialize_lab_v1_source_from_worker_stage,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import PortableWorkerSourceStage
|
||||
from k1link.observatory.recorded_jobs import RecordedExecutorIdentity
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
LAB_V1_INSTALLED_PACKAGE_CONTRACT_SCHEMA: Final = (
|
||||
"missioncore.observatory-lab-v1-installed-package-contract/v1"
|
||||
)
|
||||
|
||||
_PACKAGE_CONTRACT = Path("/opt/nodedc/package/contract.json")
|
||||
_DEFINITION_REGISTRY = Path("/opt/nodedc/package/portable-run-definitions.json")
|
||||
_DDRNET_PROFILE = Path("/opt/nodedc/package/lab-v1-eomt-ddrnet-portable-v2.json")
|
||||
_PREPARE_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "prepare"
|
||||
_EOMT_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "eomt"
|
||||
_DDRNET_STEP = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "ddrnet"
|
||||
_MAX_DOCUMENT_BYTES: Final = 2 * 1024 * 1024
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
|
||||
|
||||
class LabV1InstalledPackageStepError(RuntimeError):
|
||||
"""The installed package or one of its sealed step inputs changed."""
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
arguments = tuple(sys.argv[1:] if argv is None else argv)
|
||||
if arguments == ("prepare",):
|
||||
prepare()
|
||||
return 0
|
||||
if arguments == ("assemble",):
|
||||
assemble()
|
||||
return 0
|
||||
raise LabV1InstalledPackageStepError("LAB V1 package step is not allowlisted")
|
||||
|
||||
|
||||
def prepare() -> None:
|
||||
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "LAB V1 prepare output")
|
||||
contract = _load_contract()
|
||||
runtime_plan = _runtime_plan()
|
||||
definition = _definition(runtime_plan)
|
||||
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
|
||||
source_stage = PortableWorkerSourceStage(
|
||||
root=_real_directory(Path(INSTALLED_LAB_SOURCE_ROOT), "LAB V1 package source"),
|
||||
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "source bundle"),
|
||||
source_capability_manifest_sha256=_digest(
|
||||
runtime_plan["source_capability_manifest_sha256"],
|
||||
"source capability",
|
||||
),
|
||||
source_adapter_sha256=_digest(
|
||||
runtime_plan["source_adapter_sha256"],
|
||||
"source adapter",
|
||||
),
|
||||
)
|
||||
materialized = materialize_lab_v1_source_from_worker_stage(
|
||||
worker_stage=source_stage,
|
||||
job=job,
|
||||
definition=definition,
|
||||
output_parent=output,
|
||||
)
|
||||
camera_target = output / "camera-job"
|
||||
camera_stage_parent = materialized.camera_job_root.parent
|
||||
if camera_stage_parent.parent != materialized.root:
|
||||
raise LabV1InstalledPackageStepError(
|
||||
"materialized camera job is outside its digest-owned stage"
|
||||
)
|
||||
os.replace(materialized.camera_job_root, camera_target)
|
||||
camera_stage_parent.rmdir()
|
||||
materialized.root.rmdir()
|
||||
ddrnet_profile = _load_object(_DDRNET_PROFILE, "portable DDRNet profile")
|
||||
plan = PortableLabV1OrchestrationPlan.create_for_installed_package(
|
||||
job=job,
|
||||
definition=definition,
|
||||
source=materialized.descriptor,
|
||||
package_release_sha256=_digest(
|
||||
contract["package_release_sha256"],
|
||||
"package release",
|
||||
),
|
||||
ddrnet_runner_sha256=_digest(
|
||||
contract["ddrnet_runner_sha256"],
|
||||
"DDRNet runner",
|
||||
),
|
||||
result_assembler_sha256=_digest(
|
||||
contract["result_assembler_sha256"],
|
||||
"result assembler",
|
||||
),
|
||||
legacy_ddrnet_config=ddrnet_profile,
|
||||
)
|
||||
plan.require_executable()
|
||||
_write(output / "source-input.json", materialized.descriptor.as_dict())
|
||||
_write(output / "orchestration-plan.json", plan.as_dict())
|
||||
_write(output / "effective-ddrnet-config.json", plan.effective_ddrnet_config)
|
||||
component_images = _object(contract["component_images"], "component images")
|
||||
component_assets = _object(contract["component_assets"], "component assets")
|
||||
for component in ("eomt", "ddrnet"):
|
||||
_write(
|
||||
output / f"{component}-request.json",
|
||||
_component_request(
|
||||
component=component,
|
||||
component_image_sha256=_digest(
|
||||
component_images[component],
|
||||
f"{component} image",
|
||||
),
|
||||
source=materialized.descriptor.as_dict(),
|
||||
plan=plan,
|
||||
assets=_asset_rows(component_assets[component], component),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def assemble() -> None:
|
||||
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "LAB V1 result output")
|
||||
contract = _load_contract()
|
||||
runtime_plan = _runtime_plan()
|
||||
definition = _definition(runtime_plan)
|
||||
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
|
||||
source_input = portable_lab_v1_source_input_from_document(
|
||||
_load_object(_PREPARE_STEP / "source-input.json", "LAB V1 source input")
|
||||
)
|
||||
plan = portable_lab_v1_orchestration_plan_from_document(
|
||||
_load_object(
|
||||
_PREPARE_STEP / "orchestration-plan.json",
|
||||
"LAB V1 orchestration plan",
|
||||
),
|
||||
source_input=source_input,
|
||||
)
|
||||
if plan.release_candidate_sha256 != contract["package_release_sha256"]:
|
||||
raise LabV1InstalledPackageStepError("LAB V1 package release identity changed")
|
||||
assembly_parent = output / ".assembly"
|
||||
assembly_parent.mkdir(mode=0o700)
|
||||
package_parent = output / ".package"
|
||||
package_parent.mkdir(mode=0o700)
|
||||
assembly_result = assemble_lab_v1_result_v2(
|
||||
plan=plan,
|
||||
definition=definition,
|
||||
eomt_result_root=_real_directory(_EOMT_STEP, "EoMT step result"),
|
||||
ddrnet_result_root=_real_directory(_DDRNET_STEP, "DDRNet step result"),
|
||||
output_parent=assembly_parent,
|
||||
)
|
||||
draft = package_lab_v1_result(
|
||||
assembly=assembly_result,
|
||||
plan=plan,
|
||||
job=_SealedPackageJobView.from_job(job),
|
||||
definition=definition,
|
||||
created_at_utc=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
output_parent=package_parent,
|
||||
)
|
||||
shutil.rmtree(assembly_parent)
|
||||
for child in tuple(draft.root.iterdir()):
|
||||
os.replace(child, output / child.name)
|
||||
draft.root.rmdir()
|
||||
package_parent.rmdir()
|
||||
|
||||
|
||||
def _component_request(
|
||||
*,
|
||||
component: str,
|
||||
component_image_sha256: str,
|
||||
source: Mapping[str, object],
|
||||
plan: PortableLabV1OrchestrationPlan,
|
||||
assets: list[dict[str, object]],
|
||||
) -> dict[str, object]:
|
||||
prepared = f"{INSTALLED_LAB_STEP_INPUT_ROOT}/prepare"
|
||||
return {
|
||||
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||
"component": component,
|
||||
"component_image_sha256": component_image_sha256,
|
||||
"plan_sha256": plan.plan_sha256,
|
||||
"definition_sha256": plan.definition_sha256,
|
||||
"release_candidate_sha256": plan.release_candidate_sha256,
|
||||
"source": dict(source),
|
||||
"paths": {
|
||||
"camera_job_root": f"{prepared}/camera-job",
|
||||
"request": f"{prepared}/{component}-request.json",
|
||||
"output_root": INSTALLED_LAB_RESULT_ROOT,
|
||||
"effective_ddrnet_config": (
|
||||
f"{prepared}/effective-ddrnet-config.json"
|
||||
if component == "ddrnet"
|
||||
else None
|
||||
),
|
||||
"eomt_result_root": (
|
||||
f"{INSTALLED_LAB_STEP_INPUT_ROOT}/eomt"
|
||||
if component == "ddrnet"
|
||||
else None
|
||||
),
|
||||
"decoded_frames_root": (
|
||||
f"{INSTALLED_LAB_RESULT_ROOT}/source-frames"
|
||||
if component == "eomt"
|
||||
else f"{INSTALLED_LAB_STEP_INPUT_ROOT}/eomt/source-frames"
|
||||
),
|
||||
},
|
||||
"effective_ddrnet_config_sha256": (
|
||||
plan.effective_ddrnet_config_sha256 if component == "ddrnet" else None
|
||||
),
|
||||
"assets": assets,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
def _sealed_job(
|
||||
runtime_plan: Mapping[str, object],
|
||||
*,
|
||||
definition: PortableRunDefinition,
|
||||
contract: Mapping[str, object],
|
||||
) -> SealedObservatoryRecordedJob:
|
||||
executor = _object(contract["executor"], "package executor")
|
||||
identity = RecordedExecutorIdentity(
|
||||
release_sha256=_digest(executor["release_sha256"], "executor release"),
|
||||
image_sha256=_digest(executor["image_sha256"], "executor image"),
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||
)
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=_text(runtime_plan["job_id"], "job id"),
|
||||
request_sha256=_digest(runtime_plan["request_sha256"], "request"),
|
||||
identity_sha256=_digest(runtime_plan["identity_sha256"], "job identity"),
|
||||
submission_receipt_sha256=_digest(
|
||||
runtime_plan["submission_receipt_sha256"],
|
||||
"submission receipt",
|
||||
),
|
||||
source_session_id=_text(runtime_plan["source_session_id"], "source session"),
|
||||
source_catalog_sha256=_digest(runtime_plan["source_catalog_sha256"], "catalog"),
|
||||
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "bundle"),
|
||||
source_capability_manifest_sha256=_digest(
|
||||
runtime_plan["source_capability_manifest_sha256"],
|
||||
"source capability",
|
||||
),
|
||||
source_adapter_id=_identifier(
|
||||
runtime_plan["source_adapter_id"],
|
||||
"source adapter id",
|
||||
),
|
||||
source_adapter_version=_positive_int(
|
||||
runtime_plan["source_adapter_version"],
|
||||
"source adapter version",
|
||||
),
|
||||
source_adapter_sha256=_digest(
|
||||
runtime_plan["source_adapter_sha256"],
|
||||
"source adapter",
|
||||
),
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
executor_release_id=_identifier(executor["release_id"], "executor release id"),
|
||||
executor_identity=identity,
|
||||
model_release_ids=definition.learned_models,
|
||||
resource_profile_id=definition.resource_profile.profile_id,
|
||||
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||
claim_generation=_positive_int(
|
||||
runtime_plan["claim_generation"],
|
||||
"claim generation",
|
||||
),
|
||||
claim_claimed_at_utc=None,
|
||||
claim_expires_at_utc=None,
|
||||
claim_heartbeat_at_utc=None,
|
||||
claim_renewal_count=0,
|
||||
restart_from_zero=False,
|
||||
)
|
||||
|
||||
|
||||
def _runtime_plan() -> dict[str, object]:
|
||||
outer = _load_object(Path(INSTALLED_LAB_PLAN_PATH), "installed LAB run plan")
|
||||
_exact_keys(
|
||||
outer,
|
||||
{"schema_version", "runtime_plan", "package_id", "package_sha256", "authority"},
|
||||
"installed LAB run plan",
|
||||
)
|
||||
if (
|
||||
outer["schema_version"] != "missioncore.observatory-installed-lab-run-plan/v1"
|
||||
or outer["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise LabV1InstalledPackageStepError("installed LAB run plan changed")
|
||||
plan = _object(outer["runtime_plan"], "portable runtime plan")
|
||||
expected = {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"request_sha256",
|
||||
"identity_sha256",
|
||||
"submission_receipt_sha256",
|
||||
"claim_generation",
|
||||
"adapter_id",
|
||||
"candidate_sha256",
|
||||
"setup_id",
|
||||
"definition_id",
|
||||
"definition_version",
|
||||
"definition_sha256",
|
||||
"source_session_id",
|
||||
"source_catalog_sha256",
|
||||
"source_bundle_sha256",
|
||||
"source_capability_manifest_sha256",
|
||||
"source_adapter_id",
|
||||
"source_adapter_version",
|
||||
"source_adapter_sha256",
|
||||
"result_contract_sha256",
|
||||
"phases",
|
||||
"authority",
|
||||
}
|
||||
_exact_keys(plan, expected, "portable runtime plan")
|
||||
if (
|
||||
plan["schema_version"] != "missioncore.observatory-portable-worker-runtime-plan/v2"
|
||||
or plan["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise LabV1InstalledPackageStepError("portable runtime plan changed")
|
||||
return plan
|
||||
|
||||
|
||||
def _definition(runtime_plan: Mapping[str, object]) -> PortableRunDefinition:
|
||||
registry = PortableRunDefinitionRegistry.from_file(_DEFINITION_REGISTRY)
|
||||
return registry.resolve(
|
||||
_identifier(runtime_plan["setup_id"], "setup id"),
|
||||
_digest(runtime_plan["definition_sha256"], "definition"),
|
||||
)
|
||||
|
||||
|
||||
def _load_contract() -> dict[str, object]:
|
||||
contract = _load_object(_PACKAGE_CONTRACT, "LAB V1 package contract")
|
||||
_exact_keys(
|
||||
contract,
|
||||
{
|
||||
"schema_version",
|
||||
"package_release_sha256",
|
||||
"ddrnet_runner_sha256",
|
||||
"result_assembler_sha256",
|
||||
"executor",
|
||||
"component_images",
|
||||
"component_assets",
|
||||
"authority",
|
||||
},
|
||||
"LAB V1 package contract",
|
||||
)
|
||||
if (
|
||||
contract["schema_version"] != LAB_V1_INSTALLED_PACKAGE_CONTRACT_SCHEMA
|
||||
or contract["authority"] != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise LabV1InstalledPackageStepError("LAB V1 package contract changed")
|
||||
_exact_keys(
|
||||
_object(contract["executor"], "package executor"),
|
||||
{"release_id", "release_sha256", "image_sha256"},
|
||||
"package executor",
|
||||
)
|
||||
_exact_keys(
|
||||
_object(contract["component_images"], "component images"),
|
||||
{"eomt", "ddrnet"},
|
||||
"component images",
|
||||
)
|
||||
_exact_keys(
|
||||
_object(contract["component_assets"], "component assets"),
|
||||
{"eomt", "ddrnet"},
|
||||
"component assets",
|
||||
)
|
||||
return contract
|
||||
|
||||
|
||||
def _asset_rows(value: object, component: str) -> list[dict[str, object]]:
|
||||
if not isinstance(value, list):
|
||||
raise LabV1InstalledPackageStepError(f"{component} assets are not an array")
|
||||
rows: list[dict[str, object]] = []
|
||||
for value_row in value:
|
||||
row = _object(value_row, f"{component} asset")
|
||||
_exact_keys(
|
||||
row,
|
||||
{"asset_id", "path", "kind", "verification", "identity_sha256", "byte_length"},
|
||||
f"{component} asset",
|
||||
)
|
||||
_identifier(row["asset_id"], f"{component} asset id", dotted=True)
|
||||
_digest(row["identity_sha256"], f"{component} asset identity")
|
||||
rows.append(dict(row))
|
||||
ids = tuple(cast(str, row["asset_id"]) for row in rows)
|
||||
if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)):
|
||||
raise LabV1InstalledPackageStepError(f"{component} assets are not canonical")
|
||||
return rows
|
||||
|
||||
|
||||
def _load_object(path: Path, label: str) -> dict[str, object]:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise LabV1InstalledPackageStepError(f"{label} is unavailable")
|
||||
payload = path.read_bytes()
|
||||
if not payload or len(payload) > _MAX_DOCUMENT_BYTES:
|
||||
raise LabV1InstalledPackageStepError(f"{label} size is invalid")
|
||||
try:
|
||||
value: object = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise LabV1InstalledPackageStepError(f"{label} is invalid JSON") from exc
|
||||
return _object(value, label)
|
||||
|
||||
|
||||
def _write(path: Path, value: Mapping[str, object]) -> None:
|
||||
payload = canonical_json(value)
|
||||
path.write_bytes(payload)
|
||||
os.chmod(path, 0o400)
|
||||
if hashlib.sha256(path.read_bytes()).digest() != hashlib.sha256(payload).digest():
|
||||
raise LabV1InstalledPackageStepError("LAB V1 package document write changed")
|
||||
|
||||
|
||||
def _empty_directory(path: Path, label: str) -> Path:
|
||||
root = _real_directory(path, label)
|
||||
if any(root.iterdir()):
|
||||
raise LabV1InstalledPackageStepError(f"{label} is not empty")
|
||||
return root
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise LabV1InstalledPackageStepError(f"{label} is unavailable")
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise LabV1InstalledPackageStepError(f"{label} is not an object")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None:
|
||||
if set(value) != expected:
|
||||
raise LabV1InstalledPackageStepError(f"{label} fields changed")
|
||||
|
||||
|
||||
def _text(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise LabV1InstalledPackageStepError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _identifier(value: object, label: str, *, dotted: bool = False) -> str:
|
||||
text = _text(value, label)
|
||||
pattern = re.compile(r"^[a-z][a-z0-9.-]{2,127}$") if dotted else _IDENTIFIER
|
||||
if pattern.fullmatch(text) is None:
|
||||
raise LabV1InstalledPackageStepError(f"{label} is invalid")
|
||||
return text
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
text = _text(value, label)
|
||||
if _SHA256.fullmatch(text) is None:
|
||||
raise LabV1InstalledPackageStepError(f"{label} is invalid")
|
||||
return text
|
||||
|
||||
|
||||
def _positive_int(value: object, label: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
raise LabV1InstalledPackageStepError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except LabV1InstalledPackageStepError as exc:
|
||||
print(f"installed LAB V1 package rejected: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2) from exc
|
||||
@@ -9,153 +9,31 @@ weakening the gateway: a process-local TCP bridge binds only
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
from collections.abc import Sequence
|
||||
from contextlib import suppress
|
||||
from typing import Final
|
||||
|
||||
from k1link.observatory import m49_worker_service
|
||||
from k1link.observatory.worker_container_proxy import (
|
||||
CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS,
|
||||
CONTAINER_PROXY_COPY_BYTES,
|
||||
CONTAINER_PROXY_LISTEN_HOST,
|
||||
CONTAINER_PROXY_LISTEN_PORT,
|
||||
CONTAINER_PROXY_UPSTREAM_HOST,
|
||||
CONTAINER_PROXY_UPSTREAM_PORT,
|
||||
FixedObservatoryContainerLoopbackProxy,
|
||||
ObservatoryWorkerContainerProxyError,
|
||||
)
|
||||
|
||||
M49_CONTAINER_PROXY_LISTEN_HOST: Final = "127.0.0.1"
|
||||
M49_CONTAINER_PROXY_LISTEN_PORT: Final = 18080
|
||||
M49_CONTAINER_PROXY_UPSTREAM_HOST: Final = "host.docker.internal"
|
||||
M49_CONTAINER_PROXY_UPSTREAM_PORT: Final = 18080
|
||||
M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = 10.0
|
||||
M49_CONTAINER_PROXY_COPY_BYTES: Final = 1024 * 1024
|
||||
|
||||
|
||||
class M49WorkerContainerProxyError(RuntimeError):
|
||||
"""The fixed container loopback bridge could not be started safely."""
|
||||
|
||||
|
||||
class _ThreadedTcpServer(socketserver.ThreadingTCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
class _FixedProxyHandler(socketserver.BaseRequestHandler):
|
||||
server: _ThreadedTcpServer
|
||||
|
||||
def handle(self) -> None:
|
||||
upstream_address = getattr(self.server, "upstream_address", None)
|
||||
connect_timeout = getattr(self.server, "connect_timeout", None)
|
||||
if (
|
||||
not isinstance(upstream_address, tuple)
|
||||
or len(upstream_address) != 2
|
||||
or not isinstance(upstream_address[0], str)
|
||||
or not isinstance(upstream_address[1], int)
|
||||
or not isinstance(connect_timeout, float)
|
||||
):
|
||||
return
|
||||
try:
|
||||
upstream = socket.create_connection(
|
||||
upstream_address,
|
||||
timeout=connect_timeout,
|
||||
)
|
||||
except OSError:
|
||||
return
|
||||
with upstream:
|
||||
upstream.settimeout(None)
|
||||
client = self.request
|
||||
if not isinstance(client, socket.socket):
|
||||
return
|
||||
client.settimeout(None)
|
||||
client_to_upstream = threading.Thread(
|
||||
target=_copy_socket,
|
||||
args=(client, upstream),
|
||||
daemon=True,
|
||||
name="m49-proxy-client-to-host",
|
||||
)
|
||||
upstream_to_client = threading.Thread(
|
||||
target=_copy_socket,
|
||||
args=(upstream, client),
|
||||
daemon=True,
|
||||
name="m49-proxy-host-to-client",
|
||||
)
|
||||
client_to_upstream.start()
|
||||
upstream_to_client.start()
|
||||
client_to_upstream.join()
|
||||
upstream_to_client.join()
|
||||
|
||||
|
||||
class FixedM49ContainerLoopbackProxy:
|
||||
"""Own one bounded TCP bridge for the lifetime of the Worker process."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
listen_host: str = M49_CONTAINER_PROXY_LISTEN_HOST,
|
||||
listen_port: int = M49_CONTAINER_PROXY_LISTEN_PORT,
|
||||
upstream_host: str = M49_CONTAINER_PROXY_UPSTREAM_HOST,
|
||||
upstream_port: int = M49_CONTAINER_PROXY_UPSTREAM_PORT,
|
||||
connect_timeout: float = M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
if listen_host != M49_CONTAINER_PROXY_LISTEN_HOST:
|
||||
raise ValueError("M4.9 container proxy must bind IPv4 loopback")
|
||||
if not 0 <= listen_port <= 65_535:
|
||||
raise ValueError("M4.9 container proxy listen port is invalid")
|
||||
if not upstream_host or upstream_host != upstream_host.strip():
|
||||
raise ValueError("M4.9 container proxy upstream host is invalid")
|
||||
if not 1 <= upstream_port <= 65_535:
|
||||
raise ValueError("M4.9 container proxy upstream port is invalid")
|
||||
if not 0.05 <= connect_timeout <= 60.0:
|
||||
raise ValueError("M4.9 container proxy timeout is invalid")
|
||||
try:
|
||||
server = _ThreadedTcpServer(
|
||||
(listen_host, listen_port),
|
||||
_FixedProxyHandler,
|
||||
bind_and_activate=True,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise M49WorkerContainerProxyError(
|
||||
"M4.9 container loopback proxy could not bind"
|
||||
) from exc
|
||||
server.upstream_address = (upstream_host, upstream_port) # type: ignore[attr-defined]
|
||||
server.connect_timeout = float(connect_timeout) # type: ignore[attr-defined]
|
||||
self._server = server
|
||||
self._thread = threading.Thread(
|
||||
target=server.serve_forever,
|
||||
kwargs={"poll_interval": 0.1},
|
||||
daemon=True,
|
||||
name="m49-container-loopback-proxy",
|
||||
)
|
||||
|
||||
@property
|
||||
def listen_port(self) -> int:
|
||||
address = self._server.server_address
|
||||
if not isinstance(address, tuple) or not isinstance(address[1], int):
|
||||
raise M49WorkerContainerProxyError("M4.9 proxy address is invalid")
|
||||
return address[1]
|
||||
|
||||
def __enter__(self) -> FixedM49ContainerLoopbackProxy:
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
self._thread.join(timeout=5.0)
|
||||
if self._thread.is_alive():
|
||||
raise M49WorkerContainerProxyError(
|
||||
"M4.9 container loopback proxy did not stop"
|
||||
)
|
||||
|
||||
|
||||
def _copy_socket(source: socket.socket, destination: socket.socket) -> None:
|
||||
try:
|
||||
shutil.copyfileobj(
|
||||
source.makefile("rb", buffering=0),
|
||||
destination.makefile("wb", buffering=0),
|
||||
length=M49_CONTAINER_PROXY_COPY_BYTES,
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
with suppress(OSError):
|
||||
destination.shutdown(socket.SHUT_WR)
|
||||
M49_CONTAINER_PROXY_LISTEN_HOST: Final = CONTAINER_PROXY_LISTEN_HOST
|
||||
M49_CONTAINER_PROXY_LISTEN_PORT: Final = CONTAINER_PROXY_LISTEN_PORT
|
||||
M49_CONTAINER_PROXY_UPSTREAM_HOST: Final = CONTAINER_PROXY_UPSTREAM_HOST
|
||||
M49_CONTAINER_PROXY_UPSTREAM_PORT: Final = CONTAINER_PROXY_UPSTREAM_PORT
|
||||
M49_CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = (
|
||||
CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS
|
||||
)
|
||||
M49_CONTAINER_PROXY_COPY_BYTES: Final = CONTAINER_PROXY_COPY_BYTES
|
||||
M49WorkerContainerProxyError = ObservatoryWorkerContainerProxyError
|
||||
FixedM49ContainerLoopbackProxy = FixedObservatoryContainerLoopbackProxy
|
||||
|
||||
|
||||
def main(arguments: Sequence[str] | None = None) -> int:
|
||||
|
||||
@@ -212,11 +212,11 @@ class M49WorkerEntrypointConfiguration:
|
||||
values,
|
||||
M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
|
||||
),
|
||||
lab_v1_installation_receipt_file=_required_environment_path(
|
||||
lab_v1_installation_receipt_file=_optional_environment_path(
|
||||
values,
|
||||
LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
|
||||
),
|
||||
lab_v1_release_candidate_file=_required_environment_path(
|
||||
lab_v1_release_candidate_file=_optional_environment_path(
|
||||
values,
|
||||
LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV,
|
||||
),
|
||||
@@ -1101,6 +1101,20 @@ def _required_environment_path(values: Mapping[str, str], name: str) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def _optional_environment_path(
|
||||
values: Mapping[str, str],
|
||||
name: str,
|
||||
) -> Path | None:
|
||||
value = values.get(name)
|
||||
if value is None:
|
||||
return None
|
||||
if not value or value != value.strip():
|
||||
raise M49WorkerCompositionError(f"{name} is invalid")
|
||||
path = Path(value)
|
||||
_absolute_path(path, name)
|
||||
return path
|
||||
|
||||
|
||||
def _digest(value: str, label: str) -> None:
|
||||
if _SHA256.fullmatch(value) is None:
|
||||
raise M49WorkerCompositionError(f"{label} is invalid")
|
||||
|
||||
@@ -34,6 +34,7 @@ from k1link.compute.jobs import CameraComputeJob, validate_camera_compute_job
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PortableResultArtifact,
|
||||
PortableResultJobIdentity,
|
||||
PortableResultPackageIntegrityError,
|
||||
PortableResultPackageManifest,
|
||||
PortableResultValidationContext,
|
||||
@@ -839,52 +840,11 @@ class PortableLabV1OrchestrationPlan:
|
||||
legacy_ddrnet_config,
|
||||
source=source,
|
||||
)
|
||||
components = {item.component_id: item.sha256 for item in definition.components}
|
||||
release_assets = {asset.asset_id: asset.sha256 for asset in release.assets}
|
||||
phases = (
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="source-materialization",
|
||||
component_sha256s=(definition.source_adapter.contract_sha256,),
|
||||
input_roles=("camera-compute-job", "source-documents"),
|
||||
output_roles=("source-input-manifest",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="eomt-full-session",
|
||||
component_sha256s=tuple(
|
||||
sorted(
|
||||
(
|
||||
components["eomt-recorded-orchestrator-v1"],
|
||||
components["eomt-recorded-profile-v1"],
|
||||
components["eomt-recorded-runner-v1"],
|
||||
definition.model_manifest_sha256,
|
||||
)
|
||||
)
|
||||
),
|
||||
input_roles=("camera-compute-job",),
|
||||
output_roles=("eomt-component-result",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="ddrnet-full-session",
|
||||
component_sha256s=tuple(
|
||||
sorted(
|
||||
(
|
||||
components["ddrnet-portable-runtime-config-v2"],
|
||||
components["vegetation-mission-policy-v1"],
|
||||
components["vegetation-provider-label-map-v1"],
|
||||
release_assets["ddrnet-goose-runner"],
|
||||
definition.model_manifest_sha256,
|
||||
)
|
||||
)
|
||||
),
|
||||
input_roles=("camera-compute-job", "ddrnet-effective-config"),
|
||||
output_roles=("ddrnet-component-result",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="result-v2-assembly",
|
||||
component_sha256s=(release_assets["lab-v1-portable-contracts"],),
|
||||
input_roles=("ddrnet-component-result", "eomt-component-result"),
|
||||
output_roles=("portable-result-draft",),
|
||||
),
|
||||
phases = _portable_lab_v1_plan_phases(
|
||||
definition,
|
||||
ddrnet_runner_sha256=release_assets["ddrnet-goose-runner"],
|
||||
result_assembler_sha256=release_assets["lab-v1-portable-contracts"],
|
||||
)
|
||||
effective_sha256 = canonical_sha256(effective)
|
||||
identity = _plan_identity_document(
|
||||
@@ -920,6 +880,71 @@ class PortableLabV1OrchestrationPlan:
|
||||
plan_sha256=canonical_sha256(identity),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create_for_installed_package(
|
||||
cls,
|
||||
*,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
source: PortableLabV1SourceInput,
|
||||
package_release_sha256: str,
|
||||
ddrnet_runner_sha256: str,
|
||||
result_assembler_sha256: str,
|
||||
legacy_ddrnet_config: Mapping[str, object],
|
||||
) -> PortableLabV1OrchestrationPlan:
|
||||
"""Create a plan after the generic package registry admitted all assets."""
|
||||
|
||||
_verify_definition_and_job(definition, job)
|
||||
_verify_source_and_job(source, job, definition)
|
||||
for value, label in (
|
||||
(package_release_sha256, "installed LAB V1 package release sha256"),
|
||||
(ddrnet_runner_sha256, "installed LAB V1 DDRNet runner sha256"),
|
||||
(result_assembler_sha256, "installed LAB V1 result assembler sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
effective = build_portable_ddrnet_effective_config(
|
||||
legacy_ddrnet_config,
|
||||
source=source,
|
||||
)
|
||||
phases = _portable_lab_v1_plan_phases(
|
||||
definition,
|
||||
ddrnet_runner_sha256=ddrnet_runner_sha256,
|
||||
result_assembler_sha256=result_assembler_sha256,
|
||||
)
|
||||
effective_sha256 = canonical_sha256(effective)
|
||||
identity = _plan_identity_document(
|
||||
observatory_job_id=job.job_id,
|
||||
observatory_request_sha256=job.request_sha256,
|
||||
observatory_identity_sha256=job.identity_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
source_input_sha256=source.identity_sha256,
|
||||
release_candidate_sha256=package_release_sha256,
|
||||
effective_ddrnet_config_sha256=effective_sha256,
|
||||
phases=phases,
|
||||
release_blockers=(),
|
||||
)
|
||||
return cls(
|
||||
observatory_job_id=job.job_id,
|
||||
observatory_request_sha256=job.request_sha256,
|
||||
observatory_identity_sha256=job.identity_sha256,
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
result_contract_sha256=definition.result_contract.contract_sha256,
|
||||
source_input=source,
|
||||
release_candidate_sha256=package_release_sha256,
|
||||
effective_ddrnet_config=effective,
|
||||
effective_ddrnet_config_sha256=effective_sha256,
|
||||
phases=phases,
|
||||
blockers=(),
|
||||
plan_sha256=canonical_sha256(identity),
|
||||
)
|
||||
|
||||
@property
|
||||
def executable(self) -> bool:
|
||||
return not self.blockers
|
||||
@@ -1053,6 +1078,60 @@ def build_portable_ddrnet_effective_config(
|
||||
return copied
|
||||
|
||||
|
||||
def _portable_lab_v1_plan_phases(
|
||||
definition: PortableRunDefinition,
|
||||
*,
|
||||
ddrnet_runner_sha256: str,
|
||||
result_assembler_sha256: str,
|
||||
) -> tuple[PortableLabV1PlanPhase, ...]:
|
||||
components = {item.component_id: item.sha256 for item in definition.components}
|
||||
return (
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="source-materialization",
|
||||
component_sha256s=(definition.source_adapter.contract_sha256,),
|
||||
input_roles=("camera-compute-job", "source-documents"),
|
||||
output_roles=("source-input-manifest",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="eomt-full-session",
|
||||
component_sha256s=tuple(
|
||||
sorted(
|
||||
(
|
||||
components["eomt-recorded-orchestrator-v1"],
|
||||
components["eomt-recorded-profile-v1"],
|
||||
components["eomt-recorded-runner-v1"],
|
||||
definition.model_manifest_sha256,
|
||||
)
|
||||
)
|
||||
),
|
||||
input_roles=("camera-compute-job",),
|
||||
output_roles=("eomt-component-result",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="ddrnet-full-session",
|
||||
component_sha256s=tuple(
|
||||
sorted(
|
||||
(
|
||||
components["ddrnet-portable-runtime-config-v2"],
|
||||
components["vegetation-mission-policy-v1"],
|
||||
components["vegetation-provider-label-map-v1"],
|
||||
ddrnet_runner_sha256,
|
||||
definition.model_manifest_sha256,
|
||||
)
|
||||
)
|
||||
),
|
||||
input_roles=("camera-compute-job", "ddrnet-effective-config"),
|
||||
output_roles=("ddrnet-component-result",),
|
||||
),
|
||||
PortableLabV1PlanPhase(
|
||||
phase_id="result-v2-assembly",
|
||||
component_sha256s=(result_assembler_sha256,),
|
||||
input_roles=("ddrnet-component-result", "eomt-component-result"),
|
||||
output_roles=("portable-result-draft",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableLabV1ResultAssembly:
|
||||
root: Path
|
||||
@@ -1289,18 +1368,12 @@ def package_lab_v1_result(
|
||||
*,
|
||||
assembly: PortableLabV1ResultAssembly,
|
||||
plan: PortableLabV1OrchestrationPlan,
|
||||
job: ObservatoryRecordedJob,
|
||||
job: PortableResultJobIdentity,
|
||||
definition: PortableRunDefinition,
|
||||
created_at_utc: str,
|
||||
output_parent: Path,
|
||||
) -> PortableWorkerResultDraft:
|
||||
"""Wrap a validated assembly in the generic portable result package.
|
||||
|
||||
This step intentionally requires the durable server job because the current
|
||||
sealed Worker claim omits its submission receipt. Until that receipt is
|
||||
transported into the local adapter (or the server owns this step), release
|
||||
admission remains blocked instead of weakening the package identity.
|
||||
"""
|
||||
"""Wrap a validated assembly in the generic portable result package."""
|
||||
|
||||
_verify_plan_definition(plan, definition)
|
||||
if (
|
||||
@@ -1490,7 +1563,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
raise PortableLabV1ResultError("portable LAB V1 identity projection changed")
|
||||
|
||||
|
||||
def _source_input_from_document(
|
||||
def portable_lab_v1_source_input_from_document(
|
||||
document: Mapping[str, object],
|
||||
) -> PortableLabV1SourceInput:
|
||||
_exact_keys(
|
||||
@@ -1598,7 +1671,10 @@ def _source_input_from_document(
|
||||
)
|
||||
|
||||
|
||||
def _orchestration_plan_from_document(
|
||||
_source_input_from_document = portable_lab_v1_source_input_from_document
|
||||
|
||||
|
||||
def portable_lab_v1_orchestration_plan_from_document(
|
||||
document: Mapping[str, object],
|
||||
*,
|
||||
source_input: PortableLabV1SourceInput,
|
||||
@@ -1727,6 +1803,9 @@ def _orchestration_plan_from_document(
|
||||
return plan
|
||||
|
||||
|
||||
_orchestration_plan_from_document = portable_lab_v1_orchestration_plan_from_document
|
||||
|
||||
|
||||
def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase:
|
||||
row = _object(value, "portable plan phase")
|
||||
_exact_keys(
|
||||
@@ -1821,7 +1900,7 @@ def _validate_source_documents(
|
||||
or video.get("semantic_channel_id")
|
||||
!= requirements.camera_semantic_channel_id
|
||||
or video.get("seekable") is not True
|
||||
or camera_job.source_id != requirements.camera_source_id
|
||||
or camera_job.source_id != camera.get("public_source_id")
|
||||
or camera_job.codec_epoch != epoch.get("ordinal")
|
||||
or epoch.get("media_type") != requirements.recorded_media_type
|
||||
or init.get("sha256") != requirements.recorded_media_init_sha256
|
||||
@@ -1932,7 +2011,6 @@ def _verify_source_and_job(
|
||||
or source.source_capability_manifest_sha256
|
||||
!= job.source_capability_manifest_sha256
|
||||
or source.source_adapter_sha256 != job.source_adapter_sha256
|
||||
or source.camera_source_id != requirements.camera_source_id
|
||||
or source.calibration_sha256 != requirements.calibration_identity_sha256
|
||||
):
|
||||
raise PortableLabV1PlanError(
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Bounded recovery for verified Observatory result publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from .portable_artifact_transport import (
|
||||
PortableArtifactTransportError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from .portable_result_contract import PortableResultPublisherError
|
||||
from .portable_result_publisher import PortableObservatoryResultPublisher
|
||||
from .recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortablePublicationReconciliation:
|
||||
examined: int
|
||||
published: int
|
||||
failed: int
|
||||
deferred: int
|
||||
exhausted: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortablePublicationReconciler:
|
||||
"""Retry only durable publication outbox entries, never their compute jobs."""
|
||||
|
||||
queue: ObservatoryRecordedJobQueue
|
||||
artifact_transport: PortableObservatoryArtifactTransport
|
||||
result_publisher: PortableObservatoryResultPublisher
|
||||
clock: Callable[[], datetime] = lambda: datetime.now(UTC)
|
||||
maximum_attempts: int = 5
|
||||
retry_delays_seconds: tuple[int, ...] = (0, 30, 120, 600, 1_800)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.maximum_attempts < 1:
|
||||
raise ValueError("publication maximum attempts must be positive")
|
||||
if (
|
||||
len(self.retry_delays_seconds) != self.maximum_attempts
|
||||
or self.retry_delays_seconds != tuple(sorted(self.retry_delays_seconds))
|
||||
or any(value < 0 for value in self.retry_delays_seconds)
|
||||
):
|
||||
raise ValueError("publication retry delays are invalid")
|
||||
|
||||
def run_once(self, *, limit: int = 4) -> PortablePublicationReconciliation:
|
||||
if not 1 <= limit <= 32:
|
||||
raise ValueError("publication reconciliation limit must be within 1..32")
|
||||
now = self.clock()
|
||||
if now.tzinfo is None:
|
||||
raise ValueError("publication reconciliation clock must be timezone-aware")
|
||||
candidates = self.queue.pending_publications()[:limit]
|
||||
published = 0
|
||||
failed = 0
|
||||
deferred = 0
|
||||
exhausted = 0
|
||||
for job in candidates:
|
||||
attempts = job.publication_attempts
|
||||
if attempts >= self.maximum_attempts:
|
||||
exhausted += 1
|
||||
continue
|
||||
updated_at = _timestamp(job.updated_at_utc)
|
||||
retry_at = updated_at + timedelta(
|
||||
seconds=self.retry_delays_seconds[attempts]
|
||||
)
|
||||
if now < retry_at:
|
||||
deferred += 1
|
||||
continue
|
||||
try:
|
||||
package_root = self.artifact_transport.package_root_for_terminal(job)
|
||||
self.result_publisher.publish(job=job, package_root=package_root)
|
||||
self.queue.mark_published(job.job_id)
|
||||
published += 1
|
||||
except (PortableArtifactTransportError, PortableResultPublisherError) as exc:
|
||||
message = (" ".join(str(exc).split()) or type(exc).__name__)[:1_000]
|
||||
with suppress(ObservatoryRecordedQueueError):
|
||||
self.queue.mark_publication_failed(job.job_id, message=message)
|
||||
failed += 1
|
||||
return PortablePublicationReconciliation(
|
||||
examined=len(candidates),
|
||||
published=published,
|
||||
failed=failed,
|
||||
deferred=deferred,
|
||||
exhausted=exhausted,
|
||||
)
|
||||
|
||||
|
||||
def _timestamp(value: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ObservatoryRecordedQueueError(
|
||||
"recorded publication timestamp is invalid"
|
||||
) from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise ObservatoryRecordedQueueError(
|
||||
"recorded publication timestamp has no timezone"
|
||||
)
|
||||
return parsed
|
||||
@@ -8,7 +8,7 @@ import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Final, cast
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
@@ -16,6 +16,50 @@ from k1link.observatory.portable_run_definitions import (
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJob
|
||||
|
||||
|
||||
class PortableResultJobIdentity(Protocol):
|
||||
"""Sealed job fields required to create a portable result identity."""
|
||||
|
||||
@property
|
||||
def job_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def request_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def identity_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def submission_receipt_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def claim_generation(self) -> int: ...
|
||||
|
||||
@property
|
||||
def source_session_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def source_catalog_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def source_bundle_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def source_capability_manifest_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def source_adapter_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def source_adapter_version(self) -> int: ...
|
||||
|
||||
@property
|
||||
def source_adapter_sha256(self) -> str: ...
|
||||
|
||||
@property
|
||||
def result_id(self) -> str | None: ...
|
||||
|
||||
|
||||
PORTABLE_RESULT_PACKAGE_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-result-package/v1"
|
||||
)
|
||||
@@ -150,7 +194,7 @@ class PortableResultPackageManifest:
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
job: ObservatoryRecordedJob,
|
||||
job: PortableResultJobIdentity,
|
||||
definition: PortableRunDefinition,
|
||||
result_id: str,
|
||||
created_at_utc: str,
|
||||
@@ -415,7 +459,7 @@ class PortableResultContractValidatorRegistry:
|
||||
)
|
||||
|
||||
|
||||
def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||
def job_identity_document(job: PortableResultJobIdentity) -> dict[str, object]:
|
||||
return {
|
||||
"job_id": job.job_id,
|
||||
"request_sha256": job.request_sha256,
|
||||
@@ -425,7 +469,7 @@ def job_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||
}
|
||||
|
||||
|
||||
def source_identity_document(job: ObservatoryRecordedJob) -> dict[str, object]:
|
||||
def source_identity_document(job: PortableResultJobIdentity) -> dict[str, object]:
|
||||
return {
|
||||
"session_id": job.source_session_id,
|
||||
"catalog_sha256": job.source_catalog_sha256,
|
||||
|
||||
@@ -69,7 +69,12 @@ from k1link.observatory.source_admission import (
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY,
|
||||
)
|
||||
from k1link.sessions.models import LabSessionBinding, SessionIntegrityError, SessionSummary
|
||||
from k1link.sessions.models import (
|
||||
LabReplayCapability,
|
||||
LabSessionBinding,
|
||||
SessionIntegrityError,
|
||||
SessionSummary,
|
||||
)
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
_COPY_CHUNK_BYTES = 1024 * 1024
|
||||
@@ -157,12 +162,21 @@ class PortableObservatoryResultPublisher:
|
||||
artifact_paths=artifact_paths,
|
||||
profile=profile,
|
||||
)
|
||||
replay_capability = LabReplayCapability(
|
||||
schema_version="missioncore.observation-lab-replay-capability/v2",
|
||||
kind="portable-result-review",
|
||||
viewer_profile="portable-result",
|
||||
timeline="result-defined",
|
||||
activation="explicit",
|
||||
commands_enabled=False,
|
||||
)
|
||||
provenance = _publication_provenance(
|
||||
job=job,
|
||||
definition=definition,
|
||||
package=package,
|
||||
artifact_manifest=artifact_manifest,
|
||||
profile=profile,
|
||||
replay_capability=replay_capability,
|
||||
)
|
||||
try:
|
||||
binding = self._session_store.publish_lab_instance(
|
||||
@@ -174,7 +188,7 @@ class PortableObservatoryResultPublisher:
|
||||
result_id=cast(str, job.result_id),
|
||||
config_sha256=definition.definition_sha256,
|
||||
run_created_at_utc=job.updated_at_utc,
|
||||
replay_capability=None,
|
||||
replay_capability=replay_capability,
|
||||
provenance=provenance,
|
||||
include_recorded_media=profile.include_recorded_media,
|
||||
expected_source_catalog_sha256=job.source_catalog_sha256,
|
||||
@@ -320,6 +334,7 @@ def resolve_published_portable_calculation_profile(
|
||||
"source",
|
||||
"run_definition",
|
||||
"result_package",
|
||||
"replay_capability",
|
||||
"storage",
|
||||
"method",
|
||||
},
|
||||
@@ -354,6 +369,10 @@ def resolve_published_portable_calculation_profile(
|
||||
or binding.lab_id != profile.lab_id
|
||||
or binding.config_sha256 != definition.definition_sha256
|
||||
or binding.result_kind != definition.result_contract.result_kind
|
||||
or binding.replay_capability is None
|
||||
or binding.replay_capability.kind != "portable-result-review"
|
||||
or provenance["replay_capability"]
|
||||
!= binding.replay_capability.as_dict()
|
||||
):
|
||||
return None
|
||||
return profile.as_dict()
|
||||
@@ -712,6 +731,7 @@ def _publication_provenance(
|
||||
package: PortableResultPackageManifest,
|
||||
artifact_manifest: ArtifactManifest,
|
||||
profile: PortableCalculationProfilePolicy,
|
||||
replay_capability: LabReplayCapability,
|
||||
) -> dict[str, object]:
|
||||
result_document = next(
|
||||
artifact for artifact in package.artifacts if artifact.role == RESULT_DOCUMENT_ROLE
|
||||
@@ -732,10 +752,11 @@ def _publication_provenance(
|
||||
"result_document_sha256": result_document.sha256,
|
||||
"artifacts": [artifact.as_dict() for artifact in package.artifacts],
|
||||
},
|
||||
"replay_capability": replay_capability.as_dict(),
|
||||
"storage": {
|
||||
"mode": "central-content-addressed-artifact-store",
|
||||
"include_recorded_media": profile.include_recorded_media,
|
||||
"replay_capability": None,
|
||||
"replay_capability": replay_capability.as_dict(),
|
||||
},
|
||||
"method": _laboratory_method(job, definition),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Read-only universal viewer projection for published portable LAB results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from k1link.artifact_gateway import ArtifactGatewayError, CentralArtifactStore
|
||||
from k1link.observatory.portable_result_contract import RESULT_DOCUMENT_ROLE
|
||||
from k1link.sessions import SessionNotFoundError, SessionStore
|
||||
|
||||
PORTABLE_RESULT_VIEW_SCHEMA: Final = "missioncore.observatory-portable-result-view/v1"
|
||||
_RESULT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_MAX_RESULT_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
|
||||
|
||||
|
||||
class PortableResultViewError(RuntimeError):
|
||||
"""A published result cannot be admitted to the generic viewer."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableResultViewService:
|
||||
sessions: SessionStore
|
||||
artifacts: CentralArtifactStore
|
||||
|
||||
def read(self, result_id: str) -> dict[str, object]:
|
||||
if _RESULT_ID.fullmatch(result_id) is None:
|
||||
raise ValueError("portable result id is invalid")
|
||||
try:
|
||||
binding = self.sessions.get_lab_instance(result_id)
|
||||
except SessionNotFoundError as exc:
|
||||
raise PortableResultViewError("portable result is unavailable") from exc
|
||||
if (
|
||||
binding is None
|
||||
or binding.session_id != result_id
|
||||
or binding.result_id != result_id
|
||||
or binding.replay_capability is None
|
||||
or binding.replay_capability.kind != "portable-result-review"
|
||||
):
|
||||
raise PortableResultViewError("portable result has no viewer capability")
|
||||
provenance = binding.provenance
|
||||
package = provenance.get("result_package")
|
||||
source = provenance.get("source")
|
||||
run_definition = provenance.get("run_definition")
|
||||
calculation_profile = provenance.get("calculation_profile")
|
||||
if (
|
||||
provenance.get("schema_version")
|
||||
!= "missioncore.observatory-portable-result-publication/v1"
|
||||
or provenance.get("replay_capability")
|
||||
!= binding.replay_capability.as_dict()
|
||||
or not isinstance(package, dict)
|
||||
or not isinstance(source, dict)
|
||||
or source.get("session_id") != binding.source_session_id
|
||||
or not isinstance(run_definition, dict)
|
||||
or run_definition.get("definition_sha256") != binding.config_sha256
|
||||
or not isinstance(calculation_profile, dict)
|
||||
):
|
||||
raise PortableResultViewError("portable result provenance is invalid")
|
||||
manifest_id = package.get("artifact_manifest_id")
|
||||
package_sha256 = package.get("manifest_sha256")
|
||||
if (
|
||||
not isinstance(manifest_id, str)
|
||||
or _SHA256.fullmatch(manifest_id) is None
|
||||
or not isinstance(package_sha256, str)
|
||||
or _SHA256.fullmatch(package_sha256) is None
|
||||
):
|
||||
raise PortableResultViewError("portable result artifact identity is invalid")
|
||||
try:
|
||||
manifest = self.artifacts.read_manifest(manifest_id)
|
||||
member = manifest.member(RESULT_DOCUMENT_ROLE)
|
||||
path = self.artifacts.object_path(member.sha256)
|
||||
payload = path.read_bytes()
|
||||
except (ArtifactGatewayError, OSError) as exc:
|
||||
raise PortableResultViewError("portable result artifacts are unavailable") from exc
|
||||
if (
|
||||
manifest.artifact_type != "observatory-portable-result"
|
||||
or manifest.subject_id != result_id
|
||||
or manifest.manifest_id != manifest_id
|
||||
or manifest.metadata.get("package-sha256") != package_sha256
|
||||
or package.get("schema_version")
|
||||
!= "missioncore.observatory-portable-result-package/v1"
|
||||
or package.get("result_document_sha256") != member.sha256
|
||||
or member.media_type != "application/json"
|
||||
or not 0 < member.byte_length <= _MAX_RESULT_DOCUMENT_BYTES
|
||||
or len(payload) != member.byte_length
|
||||
or hashlib.sha256(payload).hexdigest() != member.sha256
|
||||
):
|
||||
raise PortableResultViewError("portable result artifact integrity changed")
|
||||
try:
|
||||
result_document: object = json.loads(payload.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultViewError("portable result document is invalid") from exc
|
||||
if not isinstance(result_document, dict):
|
||||
raise PortableResultViewError("portable result document must be an object")
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_VIEW_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"source_session_id": binding.source_session_id,
|
||||
"result_kind": binding.result_kind,
|
||||
"definition_sha256": binding.config_sha256,
|
||||
"viewer_capability": binding.replay_capability.as_dict(),
|
||||
"calculation_profile": calculation_profile,
|
||||
"artifact_manifest_id": manifest.manifest_id,
|
||||
"artifacts": [
|
||||
{
|
||||
"role": item.role,
|
||||
"media_type": item.media_type,
|
||||
"sha256": item.sha256,
|
||||
"byte_length": item.byte_length,
|
||||
}
|
||||
for item in manifest.members
|
||||
],
|
||||
"result_document": result_document,
|
||||
}
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
This module is deliberately only a composition boundary. It does not enable
|
||||
the Worker router, install an executor, select commands, or grant production
|
||||
authority. It binds the two admitted portable profiles to their exact result
|
||||
contracts, then constructs the local artifact transport and verified result
|
||||
publisher from server-owned dependencies.
|
||||
authority. Validators are selected by the content identity of a result
|
||||
contract, never by a LAB/setup ID, before constructing the local artifact
|
||||
transport and verified result publisher from server-owned dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,7 +19,6 @@ from typing import Final
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
M49_PORTABLE_RESULT_SCHEMA,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
@@ -27,13 +26,10 @@ from k1link.observatory.portable_artifact_transport import (
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_executor import (
|
||||
PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||
validate_lab_v1_result_v2,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PortableCalculationProfileRegistry,
|
||||
PortableResultContractValidator,
|
||||
PortableResultContractValidatorRegistration,
|
||||
PortableResultContractValidatorRegistry,
|
||||
PortableResultPublicationBlockedError,
|
||||
@@ -42,14 +38,10 @@ from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PORTABLE_M49_SETUP_ID,
|
||||
portable_calculation_profile_registry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import ObservatoryRecordedJobQueue
|
||||
@@ -161,77 +153,54 @@ class PortableObservatoryWorkerIntegration:
|
||||
supported_setup_ids: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatorSpec:
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
contract_id: str
|
||||
contract_version: int
|
||||
result_schema: str
|
||||
result_kind: str
|
||||
contract_sha256: str
|
||||
validator: PortableResultContractValidator
|
||||
|
||||
def expected_contract(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_RESULT_CONTRACT_SCHEMA,
|
||||
"contract_id": self.contract_id,
|
||||
"version": self.contract_version,
|
||||
"result_schema": self.result_schema,
|
||||
"result_kind": self.result_kind,
|
||||
"publication": "observatory",
|
||||
"contract_sha256": self.contract_sha256,
|
||||
}
|
||||
|
||||
|
||||
_VALIDATOR_SPECS: Final = (
|
||||
_ValidatorSpec(
|
||||
setup_id=PORTABLE_LAB_V1_SETUP_ID,
|
||||
definition_id="lab-v1-eomt-ddrnet-portable",
|
||||
definition_version=2,
|
||||
contract_id="recorded-eomt-ddrnet-review-v2",
|
||||
contract_version=2,
|
||||
result_schema=PORTABLE_LAB_V1_RESULT_SCHEMA,
|
||||
result_kind="recorded-perception-qualification",
|
||||
_BUILTIN_VALIDATOR_REGISTRATIONS: Final = (
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=PORTABLE_LAB_V1_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_lab_v1_result_v2,
|
||||
),
|
||||
_ValidatorSpec(
|
||||
setup_id=PORTABLE_M49_SETUP_ID,
|
||||
definition_id="m49-tgs-portable",
|
||||
definition_version=3,
|
||||
contract_id="m49-tgs-portable-review-v2",
|
||||
contract_version=2,
|
||||
result_schema=M49_PORTABLE_RESULT_SCHEMA,
|
||||
result_kind="recorded-perception-qualification",
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_m49_portable_result,
|
||||
),
|
||||
)
|
||||
_BUILTIN_VALIDATORS_BY_CONTRACT: Final = {
|
||||
registration.contract_sha256: registration.validator
|
||||
for registration in _BUILTIN_VALIDATOR_REGISTRATIONS
|
||||
}
|
||||
|
||||
|
||||
def portable_result_validator_registry(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
*,
|
||||
registrations: tuple[PortableResultContractValidatorRegistration, ...]
|
||||
| None = None,
|
||||
) -> PortableResultContractValidatorRegistry:
|
||||
"""Bind both product profiles to fixed result contracts and validators."""
|
||||
"""Select server-installed validators by exact result-contract identity."""
|
||||
|
||||
registrations: list[PortableResultContractValidatorRegistration] = []
|
||||
for spec in _VALIDATOR_SPECS:
|
||||
try:
|
||||
definition = definitions.resolve_setup(spec.setup_id)
|
||||
except PortableRunDefinitionRegistryError as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"required portable setup is unavailable: {spec.setup_id}"
|
||||
) from exc
|
||||
_verify_validator_definition(definition, spec)
|
||||
registrations.append(
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=spec.contract_sha256,
|
||||
validator=spec.validator,
|
||||
)
|
||||
)
|
||||
return PortableResultContractValidatorRegistry(tuple(registrations))
|
||||
installed = (
|
||||
_BUILTIN_VALIDATOR_REGISTRATIONS
|
||||
if registrations is None
|
||||
else registrations
|
||||
)
|
||||
by_contract = {
|
||||
registration.contract_sha256: registration
|
||||
for registration in PortableResultContractValidatorRegistry(installed).registrations
|
||||
}
|
||||
selected = tuple(
|
||||
by_contract[definition.result_contract.contract_sha256]
|
||||
for definition in definitions.definitions
|
||||
if definition.result_contract.contract_sha256 in by_contract
|
||||
)
|
||||
registry = PortableResultContractValidatorRegistry(selected)
|
||||
for definition in definitions.definitions:
|
||||
if definition.executor.ready:
|
||||
try:
|
||||
registry.resolve(definition.result_contract.contract_sha256)
|
||||
except PortableResultPublicationBlockedError as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"a ready portable definition has no exact result validator"
|
||||
) from exc
|
||||
return registry
|
||||
|
||||
|
||||
def observatory_worker_local_enabled(
|
||||
@@ -314,42 +283,43 @@ def build_portable_observatory_worker_integration(
|
||||
validators=validator_registry,
|
||||
artifact_transport=transport,
|
||||
result_publisher=publisher,
|
||||
supported_setup_ids=tuple(spec.setup_id for spec in _VALIDATOR_SPECS),
|
||||
supported_setup_ids=tuple(
|
||||
definition.setup_id
|
||||
for definition in definitions.definitions
|
||||
if any(
|
||||
registration.contract_sha256
|
||||
== definition.result_contract.contract_sha256
|
||||
for registration in validator_registry.registrations
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _verify_validator_definition(
|
||||
definition: PortableRunDefinition,
|
||||
spec: _ValidatorSpec,
|
||||
) -> None:
|
||||
if (
|
||||
definition.definition_id != spec.definition_id
|
||||
or definition.version != spec.definition_version
|
||||
or definition.result_contract.as_dict() != spec.expected_contract()
|
||||
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise PortableWorkerIntegrationError(
|
||||
f"portable result contract changed for setup: {spec.setup_id}"
|
||||
)
|
||||
|
||||
|
||||
def _verify_composition(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
profiles: PortableCalculationProfileRegistry,
|
||||
validators: PortableResultContractValidatorRegistry,
|
||||
) -> None:
|
||||
expected_validators = {spec.contract_sha256: spec.validator for spec in _VALIDATOR_SPECS}
|
||||
for definition in definitions.definitions:
|
||||
profiles.resolve(definition)
|
||||
contract_sha256 = definition.result_contract.contract_sha256
|
||||
if definition.executor.ready:
|
||||
try:
|
||||
validators.resolve(definition.result_contract.contract_sha256)
|
||||
validators.resolve(contract_sha256)
|
||||
except PortableResultPublicationBlockedError as exc:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"a ready portable definition has no exact result validator"
|
||||
) from exc
|
||||
for contract_sha256, expected in expected_validators.items():
|
||||
if validators.resolve(contract_sha256) is not expected:
|
||||
expected = _BUILTIN_VALIDATORS_BY_CONTRACT.get(contract_sha256)
|
||||
actual = next(
|
||||
(
|
||||
registration.validator
|
||||
for registration in validators.registrations
|
||||
if registration.contract_sha256 == contract_sha256
|
||||
),
|
||||
None,
|
||||
)
|
||||
if expected is not None and actual is not None and actual is not expected:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable result validator registration changed identity"
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
@@ -40,7 +40,7 @@ PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-worker-runtime-candidate/v1"
|
||||
)
|
||||
PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-worker-runtime-plan/v1"
|
||||
"missioncore.observatory-portable-worker-runtime-plan/v2"
|
||||
)
|
||||
|
||||
_MAX_REGISTRY_BYTES: Final = 256 * 1024
|
||||
@@ -63,6 +63,7 @@ type RuntimeAssetKind = Literal[
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"local-tree",
|
||||
"model-artifact",
|
||||
]
|
||||
type AssetVerificationState = Literal["matched", "missing", "mismatched"]
|
||||
@@ -116,6 +117,7 @@ class PortableWorkerAssetRequirement:
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"local-tree",
|
||||
"model-artifact",
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||
@@ -160,6 +162,18 @@ class PortableWorkerAssetRequirement:
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"model artifact requirement is incomplete"
|
||||
)
|
||||
elif self.kind == "local-tree":
|
||||
if self.byte_length is None or any(
|
||||
value is not None
|
||||
for value in (
|
||||
self.component_id,
|
||||
self.model_release_id,
|
||||
self.model_artifact_role,
|
||||
)
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"local tree requirement is incomplete"
|
||||
)
|
||||
elif any(
|
||||
value is not None
|
||||
for value in (
|
||||
@@ -478,6 +492,8 @@ def verify_local_assets(
|
||||
continue
|
||||
if requirement.kind == "container-image":
|
||||
matched = binding.file_path is None and binding.image_sha256 == requirement.sha256
|
||||
elif requirement.kind == "local-tree":
|
||||
matched = _matches_tree(requirement, binding.file_path)
|
||||
else:
|
||||
matched = _matches_file(requirement, binding.file_path)
|
||||
checks.append(
|
||||
@@ -568,12 +584,23 @@ class PortableWorkerResultDraft:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableWorkerRuntimePlan:
|
||||
job_id: str
|
||||
request_sha256: str
|
||||
identity_sha256: str
|
||||
submission_receipt_sha256: str
|
||||
claim_generation: int
|
||||
adapter_id: str
|
||||
candidate_sha256: str
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_id: str
|
||||
source_adapter_version: int
|
||||
source_adapter_sha256: str
|
||||
result_contract_sha256: str
|
||||
phases: tuple[str, ...]
|
||||
|
||||
@@ -582,19 +609,38 @@ class PortableWorkerRuntimePlan:
|
||||
for value, label in (
|
||||
(self.adapter_id, "runtime plan adapter id"),
|
||||
(self.setup_id, "runtime plan setup id"),
|
||||
(self.definition_id, "runtime plan definition id"),
|
||||
(self.source_adapter_id, "runtime plan source adapter id"),
|
||||
):
|
||||
_pattern(value, _IDENTIFIER, label)
|
||||
for value, label in (
|
||||
(self.request_sha256, "runtime plan request sha256"),
|
||||
(self.identity_sha256, "runtime plan identity sha256"),
|
||||
(self.submission_receipt_sha256, "runtime plan submission receipt sha256"),
|
||||
(self.candidate_sha256, "runtime plan candidate sha256"),
|
||||
(self.definition_sha256, "runtime plan definition sha256"),
|
||||
(self.source_catalog_sha256, "runtime plan source catalog sha256"),
|
||||
(self.source_bundle_sha256, "runtime plan source bundle sha256"),
|
||||
(
|
||||
self.source_capability_manifest_sha256,
|
||||
"runtime plan source capability sha256",
|
||||
),
|
||||
(self.source_adapter_sha256, "runtime plan source adapter sha256"),
|
||||
(self.result_contract_sha256, "runtime plan result contract sha256"),
|
||||
):
|
||||
_digest(value, label)
|
||||
_pattern(self.source_session_id, _SESSION_ID, "runtime plan source session id")
|
||||
for integer_value, label in (
|
||||
(self.claim_generation, "runtime plan claim generation"),
|
||||
(self.definition_version, "runtime plan definition version"),
|
||||
(self.source_adapter_version, "runtime plan source adapter version"),
|
||||
):
|
||||
if (
|
||||
isinstance(integer_value, bool)
|
||||
or not isinstance(integer_value, int)
|
||||
or integer_value < 1
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError(f"{label} is invalid")
|
||||
if not self.phases or len(self.phases) != len(set(self.phases)):
|
||||
raise PortableWorkerRuntimeRegistryError(
|
||||
"runtime plan phases must be non-empty and unique"
|
||||
@@ -606,12 +652,23 @@ class PortableWorkerRuntimePlan:
|
||||
return {
|
||||
"schema_version": PORTABLE_WORKER_RUNTIME_PLAN_SCHEMA,
|
||||
"job_id": self.job_id,
|
||||
"request_sha256": self.request_sha256,
|
||||
"identity_sha256": self.identity_sha256,
|
||||
"submission_receipt_sha256": self.submission_receipt_sha256,
|
||||
"claim_generation": self.claim_generation,
|
||||
"adapter_id": self.adapter_id,
|
||||
"candidate_sha256": self.candidate_sha256,
|
||||
"setup_id": self.setup_id,
|
||||
"definition_id": self.definition_id,
|
||||
"definition_version": self.definition_version,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": self.source_capability_manifest_sha256,
|
||||
"source_adapter_id": self.source_adapter_id,
|
||||
"source_adapter_version": self.source_adapter_version,
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"result_contract_sha256": self.result_contract_sha256,
|
||||
"phases": list(self.phases),
|
||||
"authority": dict(_AUTHORITY),
|
||||
@@ -687,12 +744,23 @@ class PortableWorkerExecutorAdapter:
|
||||
)
|
||||
plan = PortableWorkerRuntimePlan(
|
||||
job_id=job.job_id,
|
||||
request_sha256=job.request_sha256,
|
||||
identity_sha256=job.identity_sha256,
|
||||
submission_receipt_sha256=job.submission_receipt_sha256,
|
||||
claim_generation=job.claim_generation,
|
||||
adapter_id=self.candidate.adapter_id,
|
||||
candidate_sha256=self.candidate.candidate_sha256,
|
||||
setup_id=job.setup_id,
|
||||
definition_id=job.definition_id,
|
||||
definition_version=job.definition_version,
|
||||
definition_sha256=job.definition_sha256,
|
||||
source_session_id=job.source_session_id,
|
||||
source_catalog_sha256=job.source_catalog_sha256,
|
||||
source_bundle_sha256=job.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=job.source_capability_manifest_sha256,
|
||||
source_adapter_id=job.source_adapter_id,
|
||||
source_adapter_version=job.source_adapter_version,
|
||||
source_adapter_sha256=job.source_adapter_sha256,
|
||||
result_contract_sha256=self.candidate.result_contract_sha256,
|
||||
phases=tuple(phase.phase_id for phase in self.candidate.phases),
|
||||
)
|
||||
@@ -758,6 +826,160 @@ def _matches_file(
|
||||
return False
|
||||
|
||||
|
||||
def _matches_tree(
|
||||
requirement: PortableWorkerAssetRequirement,
|
||||
path: Path | None,
|
||||
) -> bool:
|
||||
"""Verify an install-time sealed tree without re-reading multi-GB assets."""
|
||||
|
||||
if path is None or requirement.byte_length is None:
|
||||
return False
|
||||
root = path.expanduser().absolute()
|
||||
receipt_path = root / "tree-receipt.json"
|
||||
manifest_path = root / "tree-manifest.tsv"
|
||||
try:
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
return False
|
||||
if (
|
||||
receipt_path.is_symlink()
|
||||
or not receipt_path.is_file()
|
||||
or manifest_path.is_symlink()
|
||||
or not manifest_path.is_file()
|
||||
):
|
||||
return False
|
||||
receipt_payload = receipt_path.read_bytes()
|
||||
manifest_payload = manifest_path.read_bytes()
|
||||
if not 0 < len(receipt_payload) <= 64 * 1024:
|
||||
return False
|
||||
if not 0 < len(manifest_payload) <= 16 * 1024 * 1024:
|
||||
return False
|
||||
receipt_value: object = json.loads(receipt_payload.decode("utf-8"))
|
||||
if not isinstance(receipt_value, dict):
|
||||
return False
|
||||
receipt = receipt_value
|
||||
base_keys = {
|
||||
"schema_version",
|
||||
"asset_id",
|
||||
"identity_algorithm",
|
||||
"identity_sha256",
|
||||
"file_count",
|
||||
"byte_length",
|
||||
"manifest_relative_path",
|
||||
}
|
||||
provenance_keys = {"source_image_sha256", "source_path", "binaries"}
|
||||
receipt_keys = set(receipt)
|
||||
if receipt_keys != base_keys and receipt_keys != base_keys | provenance_keys:
|
||||
return False
|
||||
if (
|
||||
receipt["schema_version"] != "missioncore.sealed-tree-runtime/v1"
|
||||
or receipt["asset_id"] != requirement.asset_id
|
||||
or receipt["identity_algorithm"]
|
||||
!= "relative-path-tab-size-tab-file-sha256-lf/v1"
|
||||
or receipt["identity_sha256"] != requirement.sha256
|
||||
or receipt["byte_length"] != requirement.byte_length
|
||||
or receipt["manifest_relative_path"] != "tree-manifest.tsv"
|
||||
or hashlib.sha256(manifest_payload).hexdigest() != requirement.sha256
|
||||
):
|
||||
return False
|
||||
rows = _sealed_tree_rows(manifest_payload)
|
||||
if receipt["file_count"] != len(rows):
|
||||
return False
|
||||
if sum(byte_length for _, byte_length, _ in rows) != requirement.byte_length:
|
||||
return False
|
||||
if receipt_keys == base_keys | provenance_keys and not _matches_tree_provenance(
|
||||
receipt,
|
||||
rows,
|
||||
):
|
||||
return False
|
||||
for relative_path, byte_length, _sha256 in rows:
|
||||
member = root.joinpath(*PurePosixPath(relative_path).parts)
|
||||
if member.is_symlink() or not member.is_file() or member.stat().st_size != byte_length:
|
||||
return False
|
||||
return True
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _matches_tree_provenance(
|
||||
receipt: Mapping[str, object],
|
||||
rows: tuple[tuple[str, int, str], ...],
|
||||
) -> bool:
|
||||
source_image_sha256 = receipt.get("source_image_sha256")
|
||||
source_path_value = receipt.get("source_path")
|
||||
binaries = receipt.get("binaries")
|
||||
if (
|
||||
not isinstance(source_image_sha256, str)
|
||||
or _SHA256.fullmatch(source_image_sha256) is None
|
||||
or not isinstance(source_path_value, str)
|
||||
or not source_path_value
|
||||
or not isinstance(binaries, dict)
|
||||
or not binaries
|
||||
):
|
||||
return False
|
||||
source_path = PurePosixPath(source_path_value)
|
||||
if not source_path.is_absolute() or ".." in source_path.parts:
|
||||
return False
|
||||
by_path = {
|
||||
relative_path: (byte_length, sha256)
|
||||
for relative_path, byte_length, sha256 in rows
|
||||
}
|
||||
for binary_id, value in binaries.items():
|
||||
if (
|
||||
not isinstance(binary_id, str)
|
||||
or _IDENTIFIER.fullmatch(binary_id) is None
|
||||
or not isinstance(value, dict)
|
||||
or set(value) != {"relative_path", "byte_length", "sha256"}
|
||||
):
|
||||
return False
|
||||
relative_path = value["relative_path"]
|
||||
byte_length = value["byte_length"]
|
||||
sha256 = value["sha256"]
|
||||
if (
|
||||
not isinstance(relative_path, str)
|
||||
or not isinstance(byte_length, int)
|
||||
or isinstance(byte_length, bool)
|
||||
or byte_length < 0
|
||||
or not isinstance(sha256, str)
|
||||
or _SHA256.fullmatch(sha256) is None
|
||||
or by_path.get(relative_path) != (byte_length, sha256)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _sealed_tree_rows(payload: bytes) -> tuple[tuple[str, int, str], ...]:
|
||||
text = payload.decode("utf-8")
|
||||
if not text.endswith("\n"):
|
||||
raise ValueError("sealed tree manifest is not newline terminated")
|
||||
rows: list[tuple[str, int, str]] = []
|
||||
previous_path: str | None = None
|
||||
for line in text.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 3:
|
||||
raise ValueError("sealed tree manifest row changed")
|
||||
relative_path, byte_length_text, sha256 = parts
|
||||
path = PurePosixPath(relative_path)
|
||||
if (
|
||||
not relative_path
|
||||
or path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or "\\" in relative_path
|
||||
or relative_path in {"tree-manifest.tsv", "tree-receipt.json"}
|
||||
or previous_path is not None
|
||||
and relative_path <= previous_path
|
||||
or _SHA256.fullmatch(sha256) is None
|
||||
):
|
||||
raise ValueError("sealed tree manifest identity changed")
|
||||
byte_length = int(byte_length_text)
|
||||
if byte_length < 0 or str(byte_length) != byte_length_text:
|
||||
raise ValueError("sealed tree manifest byte length changed")
|
||||
rows.append((relative_path, byte_length, sha256))
|
||||
previous_path = relative_path
|
||||
if not rows:
|
||||
raise ValueError("sealed tree manifest is empty")
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def _candidate(value: object) -> PortableWorkerRuntimeCandidate:
|
||||
row = _object(value, "runtime candidate")
|
||||
_exact_keys(
|
||||
@@ -844,6 +1066,7 @@ def _asset(value: object) -> PortableWorkerAssetRequirement:
|
||||
"container-image",
|
||||
"definition-component",
|
||||
"local-file",
|
||||
"local-tree",
|
||||
"model-artifact",
|
||||
):
|
||||
raise PortableWorkerRuntimeRegistryError("runtime asset kind is invalid")
|
||||
|
||||
@@ -44,6 +44,10 @@ OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = "missioncore.observatory-live-k1-
|
||||
RECORDED_JOB_DATABASE_NAME: Final = "observatory-recorded-jobs.sqlite3"
|
||||
MAX_RECORDED_JOBS: Final = 10_000
|
||||
MAX_RECORDED_CLAIM_RECEIPTS: Final = 50_000
|
||||
# Every capability expands to four SQLite bind parameters. Keeping the public
|
||||
# bound at 128 stays comfortably below SQLite's traditional 999-variable limit
|
||||
# even when the runtime was compiled with conservative defaults.
|
||||
MAX_RECORDED_EXECUTOR_CAPABILITIES: Final = 128
|
||||
MAX_LIVE_LEASES: Final = 10_000
|
||||
MAX_RECORDED_JOB_STORAGE_BYTES: Final = 128 * 1024 * 1024
|
||||
RECORDED_JOB_SQLITE_LOCK_TIMEOUT_SECONDS: Final = 0.1
|
||||
@@ -67,6 +71,7 @@ type RecordedJobState = Literal[
|
||||
"reconciliation-required",
|
||||
]
|
||||
type CheckpointPolicy = Literal["cooperative", "non-checkpointable"]
|
||||
type PublicationState = Literal["not-required", "pending", "failed", "published"]
|
||||
type LiveLeaseState = Literal["pending", "active", "completed", "failed", "cancelled"]
|
||||
type LiveTerminalOutcome = Literal["completed", "failed", "cancelled"]
|
||||
|
||||
@@ -139,6 +144,12 @@ CREATE TABLE IF NOT EXISTS observatory_recorded_jobs (
|
||||
terminal_code TEXT,
|
||||
terminal_message TEXT,
|
||||
terminal_claim_token_sha256 TEXT,
|
||||
publication_state TEXT NOT NULL DEFAULT 'not-required'
|
||||
CHECK (publication_state IN ('not-required', 'pending', 'failed', 'published')),
|
||||
publication_attempts INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (publication_attempts >= 0),
|
||||
publication_error TEXT,
|
||||
published_at_utc TEXT,
|
||||
created_at_utc TEXT NOT NULL,
|
||||
updated_at_utc TEXT NOT NULL
|
||||
);
|
||||
@@ -256,6 +267,33 @@ class ObservatoryRecordedPreemptionError(ObservatoryRecordedQueueError):
|
||||
"""The scheduler could not prove immediate release for live K1."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, order=True)
|
||||
class RecordedExecutorIdentity:
|
||||
"""Path-free executor capability shared by scheduling and Worker code."""
|
||||
|
||||
release_sha256: str
|
||||
image_sha256: str
|
||||
model_manifest_sha256: str
|
||||
resource_profile_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for value, label in (
|
||||
(self.release_sha256, "executor release sha256"),
|
||||
(self.image_sha256, "executor image sha256"),
|
||||
(self.model_manifest_sha256, "model manifest sha256"),
|
||||
(self.resource_profile_sha256, "resource profile sha256"),
|
||||
):
|
||||
_validate_digest(value, label)
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"release_sha256": self.release_sha256,
|
||||
"image_sha256": self.image_sha256,
|
||||
"model_manifest_sha256": self.model_manifest_sha256,
|
||||
"resource_profile_sha256": self.resource_profile_sha256,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedRunDefinition:
|
||||
"""Server-owned executable identity; it contains no executable text or path."""
|
||||
@@ -432,6 +470,10 @@ class ObservatoryRecordedJob:
|
||||
terminal_code: str | None
|
||||
terminal_message: str | None
|
||||
terminal_claim_token_sha256: str | None
|
||||
publication_state: PublicationState
|
||||
publication_attempts: int
|
||||
publication_error: str | None
|
||||
published_at_utc: str | None
|
||||
created_at_utc: str
|
||||
updated_at_utc: str
|
||||
priority_class: Literal["recorded"] = "recorded"
|
||||
@@ -557,6 +599,59 @@ class ObservatoryRecordedJob:
|
||||
_SHA256,
|
||||
"terminal claim token sha256",
|
||||
)
|
||||
if self.publication_state not in (
|
||||
"not-required",
|
||||
"pending",
|
||||
"failed",
|
||||
"published",
|
||||
):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job publication state is invalid"
|
||||
)
|
||||
if self.publication_attempts < 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job publication attempt count is invalid"
|
||||
)
|
||||
if self.publication_error is not None:
|
||||
_validate_text(
|
||||
self.publication_error,
|
||||
"recorded-job publication error",
|
||||
max_length=1_000,
|
||||
)
|
||||
if self.published_at_utc is not None:
|
||||
_validate_timestamp(self.published_at_utc, "recorded-job publication timestamp")
|
||||
publication_shape = {
|
||||
"not-required": (
|
||||
self.publication_attempts == 0
|
||||
and self.publication_error is None
|
||||
and self.published_at_utc is None
|
||||
),
|
||||
"pending": (
|
||||
self.publication_error is None and self.published_at_utc is None
|
||||
),
|
||||
"failed": (
|
||||
self.publication_attempts >= 1
|
||||
and self.publication_error is not None
|
||||
and self.published_at_utc is None
|
||||
),
|
||||
"published": (
|
||||
self.publication_attempts >= 1
|
||||
and self.publication_error is None
|
||||
and self.published_at_utc is not None
|
||||
),
|
||||
}
|
||||
if not publication_shape[self.publication_state]:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job publication receipt is inconsistent"
|
||||
)
|
||||
if self.publication_state != "not-required" and (
|
||||
self.state != "succeeded"
|
||||
or self.result_id is None
|
||||
or self.result_sha256 is None
|
||||
):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job publication lifecycle has no execution result"
|
||||
)
|
||||
_validate_timestamp(self.created_at_utc, "created timestamp")
|
||||
_validate_timestamp(self.updated_at_utc, "updated timestamp")
|
||||
|
||||
@@ -630,6 +725,12 @@ class ObservatoryRecordedJob:
|
||||
if self.terminal_code is None
|
||||
else {"code": self.terminal_code, "message": self.terminal_message}
|
||||
),
|
||||
"publication": {
|
||||
"state": self.publication_state,
|
||||
"attempts": self.publication_attempts,
|
||||
"error": self.publication_error,
|
||||
"published_at_utc": self.published_at_utc,
|
||||
},
|
||||
"created_at_utc": self.created_at_utc,
|
||||
"updated_at_utc": self.updated_at_utc,
|
||||
"authority": dict(_AUTHORITY),
|
||||
@@ -1192,12 +1293,23 @@ class ObservatoryRecordedJobQueue:
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None = None,
|
||||
) -> ObservatoryRecordedClaim | None:
|
||||
"""Claim one recorded job atomically; even an empty claim is idempotent."""
|
||||
"""Claim one compatible job atomically; even an empty claim is idempotent.
|
||||
|
||||
``None`` preserves the legacy v1 claim semantics during rollout. A
|
||||
concrete tuple is the capability-aware v2 contract; an empty tuple
|
||||
deliberately claims nothing.
|
||||
"""
|
||||
|
||||
_validate_pattern(claimant_id, _IDENTIFIER, "claimant id")
|
||||
_validate_pattern(claim_request_id, _IDEMPOTENCY_KEY, "claim request id")
|
||||
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
||||
capabilities = _canonical_executor_capabilities(supported_executor_identities)
|
||||
request_sha256 = _claim_request_sha256(
|
||||
claimant_id,
|
||||
claim_request_id,
|
||||
supported_executor_identities=capabilities,
|
||||
)
|
||||
with self._transaction() as connection:
|
||||
now = self._timestamp()
|
||||
self._recover_stale_claims(connection, now=now)
|
||||
@@ -1226,11 +1338,10 @@ class ObservatoryRecordedJobQueue:
|
||||
"LIMIT 1"
|
||||
).fetchone()
|
||||
if active_owner is None:
|
||||
row = connection.execute(
|
||||
"SELECT job_id FROM observatory_recorded_jobs "
|
||||
"WHERE state = 'queued' "
|
||||
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
|
||||
).fetchone()
|
||||
row = self._next_compatible_queued_job(
|
||||
connection,
|
||||
supported_executor_identities=capabilities,
|
||||
)
|
||||
if row is None:
|
||||
connection.execute(
|
||||
"INSERT INTO observatory_recorded_claim_receipts "
|
||||
@@ -1289,6 +1400,44 @@ class ObservatoryRecordedJobQueue:
|
||||
job=self._get_job(connection, job_id),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _next_compatible_queued_job(
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None,
|
||||
) -> sqlite3.Row | None:
|
||||
if supported_executor_identities is None:
|
||||
row: sqlite3.Row | None = connection.execute(
|
||||
"SELECT job_id FROM observatory_recorded_jobs "
|
||||
"WHERE state = 'queued' "
|
||||
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1"
|
||||
).fetchone()
|
||||
return row
|
||||
if not supported_executor_identities:
|
||||
return None
|
||||
predicates = " OR ".join(
|
||||
"(executor_release_sha256 = ? AND executor_image_sha256 = ? "
|
||||
"AND model_manifest_sha256 = ? AND resource_profile_sha256 = ?)"
|
||||
for _identity in supported_executor_identities
|
||||
)
|
||||
parameters = tuple(
|
||||
value
|
||||
for identity in supported_executor_identities
|
||||
for value in (
|
||||
identity.release_sha256,
|
||||
identity.image_sha256,
|
||||
identity.model_manifest_sha256,
|
||||
identity.resource_profile_sha256,
|
||||
)
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT job_id FROM observatory_recorded_jobs "
|
||||
f"WHERE state = 'queued' AND ({predicates}) "
|
||||
"ORDER BY priority_rank, created_at_utc, job_id LIMIT 1",
|
||||
parameters,
|
||||
).fetchone()
|
||||
return row
|
||||
|
||||
def renew_claim(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -1653,6 +1802,96 @@ class ObservatoryRecordedJobQueue:
|
||||
terminal_message="Recorded result was sealed by the Worker.",
|
||||
)
|
||||
|
||||
def complete_for_publication(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
claim_token: str,
|
||||
result_id: str,
|
||||
result_sha256: str,
|
||||
) -> ObservatoryRecordedJob:
|
||||
"""Seal execution and atomically enqueue its verified publication."""
|
||||
|
||||
_validate_pattern(result_id, _SESSION_ID, "result id")
|
||||
_validate_digest(result_sha256, "result sha256")
|
||||
return self._terminal_job_transition(
|
||||
job_id,
|
||||
claim_token=claim_token,
|
||||
state="succeeded",
|
||||
result_id=result_id,
|
||||
result_sha256=result_sha256,
|
||||
terminal_code="result-sealed",
|
||||
terminal_message="Recorded result was sealed by the Worker.",
|
||||
publication_state="pending",
|
||||
)
|
||||
|
||||
def mark_publication_failed(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
message: str,
|
||||
) -> ObservatoryRecordedJob:
|
||||
"""Record one failed outbox attempt without losing the execution result."""
|
||||
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_text(message, "recorded publication error", max_length=1_000)
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
if job.state != "succeeded" or job.publication_state not in {
|
||||
"pending",
|
||||
"failed",
|
||||
}:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded result is not awaiting publication"
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs "
|
||||
"SET publication_state = 'failed', "
|
||||
"publication_attempts = publication_attempts + 1, "
|
||||
"publication_error = ?, published_at_utc = NULL, "
|
||||
"updated_at_utc = ? WHERE job_id = ?",
|
||||
(message, self._timestamp(), job_id),
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def mark_published(self, job_id: str) -> ObservatoryRecordedJob:
|
||||
"""Acknowledge one idempotently published outbox entry."""
|
||||
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
if job.state != "succeeded":
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded execution has not succeeded"
|
||||
)
|
||||
if job.publication_state == "published":
|
||||
return job
|
||||
if job.publication_state not in {"pending", "failed"}:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded result has no publication outbox entry"
|
||||
)
|
||||
now = self._timestamp()
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs "
|
||||
"SET publication_state = 'published', "
|
||||
"publication_attempts = publication_attempts + 1, "
|
||||
"publication_error = NULL, published_at_utc = ?, "
|
||||
"updated_at_utc = ? WHERE job_id = ?",
|
||||
(now, now, job_id),
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def pending_publications(self) -> tuple[ObservatoryRecordedJob, ...]:
|
||||
"""Return durable outbox entries in deterministic retry order."""
|
||||
|
||||
with self._read_connection() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs "
|
||||
"WHERE publication_state IN ('pending', 'failed') "
|
||||
"ORDER BY created_at_utc, job_id"
|
||||
).fetchall()
|
||||
return tuple(_job_from_row(row) for row in rows)
|
||||
|
||||
def fail(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -2038,6 +2277,7 @@ class ObservatoryRecordedJobQueue:
|
||||
result_sha256: str | None,
|
||||
terminal_code: str,
|
||||
terminal_message: str,
|
||||
publication_state: Literal["not-required", "pending"] = "not-required",
|
||||
) -> ObservatoryRecordedJob:
|
||||
_validate_pattern(job_id, _JOB_ID, "recorded job id")
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
@@ -2053,6 +2293,11 @@ class ObservatoryRecordedJobQueue:
|
||||
and job.terminal_code == terminal_code
|
||||
and job.terminal_message == terminal_message
|
||||
and job.terminal_claim_token_sha256 == token_sha256
|
||||
and (
|
||||
job.publication_state == "not-required"
|
||||
if publication_state == "not-required"
|
||||
else job.publication_state in {"pending", "failed", "published"}
|
||||
)
|
||||
)
|
||||
if exact_replay:
|
||||
return job
|
||||
@@ -2091,7 +2336,9 @@ class ObservatoryRecordedJobQueue:
|
||||
"terminal_claim_token_sha256 = ?, active_claim_token = NULL, "
|
||||
"active_claimant_id = NULL, claimed_at_utc = NULL, "
|
||||
"claim_expires_at_utc = NULL, claim_heartbeat_at_utc = NULL, "
|
||||
"claim_renewal_count = 0, updated_at_utc = ? WHERE job_id = ?",
|
||||
"claim_renewal_count = 0, publication_state = ?, "
|
||||
"publication_attempts = 0, publication_error = NULL, "
|
||||
"published_at_utc = NULL, updated_at_utc = ? WHERE job_id = ?",
|
||||
(
|
||||
state,
|
||||
result_id,
|
||||
@@ -2099,6 +2346,7 @@ class ObservatoryRecordedJobQueue:
|
||||
terminal_code,
|
||||
terminal_message,
|
||||
token_sha256,
|
||||
publication_state,
|
||||
now,
|
||||
job_id,
|
||||
),
|
||||
@@ -2387,6 +2635,7 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._connect() as connection:
|
||||
connection.executescript(_SCHEMA_SQL)
|
||||
self._migrate_claim_lease_schema(connection)
|
||||
self._migrate_publication_schema(connection)
|
||||
self._validate_schema(connection)
|
||||
self._validate_existing_capacity(connection)
|
||||
connection.commit()
|
||||
@@ -2403,7 +2652,7 @@ class ObservatoryRecordedJobQueue:
|
||||
|
||||
def _validate_schema(self, connection: sqlite3.Connection) -> None:
|
||||
expected = {
|
||||
"observatory_recorded_jobs": 46,
|
||||
"observatory_recorded_jobs": 50,
|
||||
"observatory_recorded_claim_receipts": 6,
|
||||
"observatory_recorded_reconciliations": 18,
|
||||
"observatory_live_leases": 13,
|
||||
@@ -2508,6 +2757,35 @@ class ObservatoryRecordedJobQueue:
|
||||
"legacy active claim is stored in an invalid state"
|
||||
)
|
||||
|
||||
def _migrate_publication_schema(self, connection: sqlite3.Connection) -> None:
|
||||
"""Add the durable result-publication outbox to existing queues."""
|
||||
|
||||
columns = {
|
||||
str(row["name"])
|
||||
for row in connection.execute(
|
||||
"SELECT name FROM pragma_table_info('observatory_recorded_jobs')"
|
||||
).fetchall()
|
||||
}
|
||||
additions = (
|
||||
(
|
||||
"publication_state",
|
||||
"TEXT NOT NULL DEFAULT 'not-required' "
|
||||
"CHECK (publication_state IN "
|
||||
"('not-required', 'pending', 'failed', 'published'))",
|
||||
),
|
||||
(
|
||||
"publication_attempts",
|
||||
"INTEGER NOT NULL DEFAULT 0 CHECK (publication_attempts >= 0)",
|
||||
),
|
||||
("publication_error", "TEXT"),
|
||||
("published_at_utc", "TEXT"),
|
||||
)
|
||||
for name, definition in additions:
|
||||
if name not in columns:
|
||||
connection.execute(
|
||||
f"ALTER TABLE observatory_recorded_jobs ADD COLUMN {name} {definition}"
|
||||
)
|
||||
|
||||
def _validate_existing_capacity(self, connection: sqlite3.Connection) -> None:
|
||||
for table, limit, label in (
|
||||
("observatory_recorded_jobs", self._max_jobs, "recorded job"),
|
||||
@@ -2724,6 +3002,10 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
terminal_code=row["terminal_code"],
|
||||
terminal_message=row["terminal_message"],
|
||||
terminal_claim_token_sha256=row["terminal_claim_token_sha256"],
|
||||
publication_state=row["publication_state"],
|
||||
publication_attempts=row["publication_attempts"],
|
||||
publication_error=row["publication_error"],
|
||||
published_at_utc=row["published_at_utc"],
|
||||
created_at_utc=row["created_at_utc"],
|
||||
updated_at_utc=row["updated_at_utc"],
|
||||
)
|
||||
@@ -2801,14 +3083,37 @@ def _submission_receipt_sha256(
|
||||
)
|
||||
|
||||
|
||||
def _claim_request_sha256(claimant_id: str, claim_request_id: str) -> str:
|
||||
return _sha256(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": claimant_id,
|
||||
}
|
||||
)
|
||||
def _claim_request_sha256(
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
*,
|
||||
supported_executor_identities: tuple[RecordedExecutorIdentity, ...] | None = None,
|
||||
) -> str:
|
||||
document: dict[str, object] = {
|
||||
"schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": claimant_id,
|
||||
}
|
||||
if supported_executor_identities is not None:
|
||||
document["supported_executor_identities"] = [
|
||||
identity.as_dict() for identity in supported_executor_identities
|
||||
]
|
||||
return _sha256(document)
|
||||
|
||||
|
||||
def _canonical_executor_capabilities(
|
||||
identities: tuple[RecordedExecutorIdentity, ...] | None,
|
||||
) -> tuple[RecordedExecutorIdentity, ...] | None:
|
||||
if identities is None:
|
||||
return None
|
||||
if len(identities) > MAX_RECORDED_EXECUTOR_CAPABILITIES:
|
||||
raise ValueError("too many recorded executor capabilities")
|
||||
if any(not isinstance(identity, RecordedExecutorIdentity) for identity in identities):
|
||||
raise ValueError("recorded executor capability is invalid")
|
||||
canonical = tuple(sorted(identities))
|
||||
if len(canonical) != len(set(canonical)):
|
||||
raise ValueError("recorded executor capabilities must be unique")
|
||||
return canonical
|
||||
|
||||
|
||||
def _recorded_job_states() -> frozenset[str]:
|
||||
|
||||
@@ -29,6 +29,7 @@ from k1link.observatory.recorded_jobs import (
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
@@ -53,6 +54,7 @@ type WorkerCycleState = Literal[
|
||||
"failed",
|
||||
"rejected",
|
||||
"lease-lost",
|
||||
"publication-pending",
|
||||
]
|
||||
type RecordedJobWireState = Literal[
|
||||
"accepted",
|
||||
@@ -97,24 +99,7 @@ class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError):
|
||||
"""No local adapter matches the exact sealed executor identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorIdentity:
|
||||
"""The only identity that may select executable Worker code."""
|
||||
|
||||
release_sha256: str
|
||||
image_sha256: str
|
||||
model_manifest_sha256: str
|
||||
resource_profile_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for label, value in (
|
||||
("executor release", self.release_sha256),
|
||||
("executor image", self.image_sha256),
|
||||
("model manifest", self.model_manifest_sha256),
|
||||
("resource profile", self.resource_profile_sha256),
|
||||
):
|
||||
if re.fullmatch(_SHA256_PATTERN, value) is None:
|
||||
raise ValueError(f"{label} SHA-256 is invalid")
|
||||
ObservatoryWorkerExecutorIdentity = RecordedExecutorIdentity
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -201,6 +186,12 @@ class ObservatoryWorkerExecutorRegistry:
|
||||
"exact executor identity is not locally allowlisted"
|
||||
)
|
||||
|
||||
@property
|
||||
def supported_identities(self) -> tuple[ObservatoryWorkerExecutorIdentity, ...]:
|
||||
"""Canonical capability snapshot sent with every claim request."""
|
||||
|
||||
return tuple(sorted(registration.identity for registration in self.registrations))
|
||||
|
||||
|
||||
class ObservatoryWorkerTransport(Protocol):
|
||||
"""State-transition port implemented by HTTP, IPC, or a test transport."""
|
||||
@@ -210,6 +201,7 @@ class ObservatoryWorkerTransport(Protocol):
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...],
|
||||
) -> Mapping[str, object] | None: ...
|
||||
|
||||
def start(
|
||||
@@ -332,6 +324,13 @@ class _ClaimLeasePayload(_StrictPayload):
|
||||
renewal_count: int = Field(ge=0)
|
||||
|
||||
|
||||
class _PublicationPayload(_StrictPayload):
|
||||
state: Literal["not-required", "pending", "failed", "published"]
|
||||
attempts: int = Field(ge=0)
|
||||
error: str | None
|
||||
published_at_utc: Timestamp | None
|
||||
|
||||
|
||||
class _RecordedJobPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
||||
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
||||
@@ -356,6 +355,7 @@ class _RecordedJobPayload(_StrictPayload):
|
||||
claim_lease: _ClaimLeasePayload | None
|
||||
result: _ResultPayload | None
|
||||
terminal: _TerminalPayload | None
|
||||
publication: _PublicationPayload
|
||||
created_at_utc: Timestamp
|
||||
updated_at_utc: Timestamp
|
||||
authority: _AuthorityPayload
|
||||
@@ -423,6 +423,7 @@ class ObservatoryWorkerAgent:
|
||||
payload = self._transport.claim_next(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
claim_request_id=claim_request_id,
|
||||
supported_executor_identities=self._executors.supported_identities,
|
||||
)
|
||||
if payload is None:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
@@ -430,7 +431,11 @@ class ObservatoryWorkerAgent:
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
try:
|
||||
claim = _validate_claim(payload, claim_request_id=claim_request_id)
|
||||
claim = _validate_claim(
|
||||
payload,
|
||||
claim_request_id=claim_request_id,
|
||||
supported_executor_identities=self._executors.supported_identities,
|
||||
)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="rejected",
|
||||
@@ -554,7 +559,11 @@ class ObservatoryWorkerAgent:
|
||||
"Worker success acknowledgement changed result identity"
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="succeeded",
|
||||
state=(
|
||||
"publication-pending"
|
||||
if succeeded.publication.state in {"pending", "failed"}
|
||||
else "succeeded"
|
||||
),
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
result_id=result.result_id,
|
||||
@@ -651,6 +660,7 @@ def _validate_claim(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...],
|
||||
) -> _ValidatedClaim:
|
||||
try:
|
||||
claim = _RecordedClaimPayload.model_validate(dict(payload))
|
||||
@@ -663,6 +673,10 @@ def _validate_claim(
|
||||
"schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": WORKER_006_CONTOUR_ID,
|
||||
"supported_executor_identities": [
|
||||
identity.as_dict()
|
||||
for identity in sorted(supported_executor_identities)
|
||||
],
|
||||
}
|
||||
)
|
||||
if claim.request_sha256 != expected_claim_request_sha256:
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Fixed loopback bridge for Observatory Worker containers on Docker Desktop.
|
||||
|
||||
The authenticated Worker HTTP gateway accepts plaintext only on a loopback
|
||||
URL. Docker Desktop exposes the Windows host as ``host.docker.internal``, so a
|
||||
container-owned bridge binds one fixed loopback socket and forwards it to the
|
||||
Mac-owned reverse SSH tunnel on the Worker host. No address, port, credential,
|
||||
or destination is supplied by a queued job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
from contextlib import suppress
|
||||
from typing import Final
|
||||
|
||||
CONTAINER_PROXY_LISTEN_HOST: Final = "127.0.0.1"
|
||||
CONTAINER_PROXY_LISTEN_PORT: Final = 18080
|
||||
CONTAINER_PROXY_UPSTREAM_HOST: Final = "host.docker.internal"
|
||||
CONTAINER_PROXY_UPSTREAM_PORT: Final = 18080
|
||||
CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS: Final = 10.0
|
||||
CONTAINER_PROXY_COPY_BYTES: Final = 1024 * 1024
|
||||
|
||||
|
||||
class ObservatoryWorkerContainerProxyError(RuntimeError):
|
||||
"""The fixed container loopback bridge could not be started safely."""
|
||||
|
||||
|
||||
class _ThreadedTcpServer(socketserver.ThreadingTCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
class _FixedProxyHandler(socketserver.BaseRequestHandler):
|
||||
server: _ThreadedTcpServer
|
||||
|
||||
def handle(self) -> None:
|
||||
upstream_address = getattr(self.server, "upstream_address", None)
|
||||
connect_timeout = getattr(self.server, "connect_timeout", None)
|
||||
if (
|
||||
not isinstance(upstream_address, tuple)
|
||||
or len(upstream_address) != 2
|
||||
or not isinstance(upstream_address[0], str)
|
||||
or not isinstance(upstream_address[1], int)
|
||||
or not isinstance(connect_timeout, float)
|
||||
):
|
||||
return
|
||||
try:
|
||||
upstream = socket.create_connection(
|
||||
upstream_address,
|
||||
timeout=connect_timeout,
|
||||
)
|
||||
except OSError:
|
||||
return
|
||||
with upstream:
|
||||
upstream.settimeout(None)
|
||||
client = self.request
|
||||
if not isinstance(client, socket.socket):
|
||||
return
|
||||
client.settimeout(None)
|
||||
client_to_upstream = threading.Thread(
|
||||
target=_copy_socket,
|
||||
args=(client, upstream),
|
||||
daemon=True,
|
||||
name="observatory-proxy-client-to-host",
|
||||
)
|
||||
upstream_to_client = threading.Thread(
|
||||
target=_copy_socket,
|
||||
args=(upstream, client),
|
||||
daemon=True,
|
||||
name="observatory-proxy-host-to-client",
|
||||
)
|
||||
client_to_upstream.start()
|
||||
upstream_to_client.start()
|
||||
client_to_upstream.join()
|
||||
upstream_to_client.join()
|
||||
|
||||
|
||||
class FixedObservatoryContainerLoopbackProxy:
|
||||
"""Own one bounded TCP bridge for the lifetime of a Worker process."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
listen_host: str = CONTAINER_PROXY_LISTEN_HOST,
|
||||
listen_port: int = CONTAINER_PROXY_LISTEN_PORT,
|
||||
upstream_host: str = CONTAINER_PROXY_UPSTREAM_HOST,
|
||||
upstream_port: int = CONTAINER_PROXY_UPSTREAM_PORT,
|
||||
connect_timeout: float = CONTAINER_PROXY_CONNECT_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
if listen_host != CONTAINER_PROXY_LISTEN_HOST:
|
||||
raise ValueError("Observatory container proxy must bind IPv4 loopback")
|
||||
if not 0 <= listen_port <= 65_535:
|
||||
raise ValueError("Observatory container proxy listen port is invalid")
|
||||
if upstream_host != CONTAINER_PROXY_UPSTREAM_HOST and upstream_host != "127.0.0.1":
|
||||
raise ValueError("Observatory container proxy upstream host is invalid")
|
||||
if not 1 <= upstream_port <= 65_535:
|
||||
raise ValueError("Observatory container proxy upstream port is invalid")
|
||||
if not 0.05 <= connect_timeout <= 60.0:
|
||||
raise ValueError("Observatory container proxy timeout is invalid")
|
||||
try:
|
||||
server = _ThreadedTcpServer(
|
||||
(listen_host, listen_port),
|
||||
_FixedProxyHandler,
|
||||
bind_and_activate=True,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise ObservatoryWorkerContainerProxyError(
|
||||
"Observatory container loopback proxy could not bind"
|
||||
) from exc
|
||||
server.upstream_address = (upstream_host, upstream_port) # type: ignore[attr-defined]
|
||||
server.connect_timeout = float(connect_timeout) # type: ignore[attr-defined]
|
||||
self._server = server
|
||||
self._thread = threading.Thread(
|
||||
target=server.serve_forever,
|
||||
kwargs={"poll_interval": 0.1},
|
||||
daemon=True,
|
||||
name="observatory-container-loopback-proxy",
|
||||
)
|
||||
|
||||
@property
|
||||
def listen_port(self) -> int:
|
||||
address = self._server.server_address
|
||||
if not isinstance(address, tuple) or not isinstance(address[1], int):
|
||||
raise ObservatoryWorkerContainerProxyError(
|
||||
"Observatory proxy address is invalid"
|
||||
)
|
||||
return address[1]
|
||||
|
||||
def __enter__(self) -> FixedObservatoryContainerLoopbackProxy:
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
self._thread.join(timeout=5.0)
|
||||
if self._thread.is_alive():
|
||||
raise ObservatoryWorkerContainerProxyError(
|
||||
"Observatory container loopback proxy did not stop"
|
||||
)
|
||||
|
||||
|
||||
def _copy_socket(source: socket.socket, destination: socket.socket) -> None:
|
||||
try:
|
||||
shutil.copyfileobj(
|
||||
source.makefile("rb", buffering=0),
|
||||
destination.makefile("wb", buffering=0),
|
||||
length=CONTAINER_PROXY_COPY_BYTES,
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
with suppress(OSError):
|
||||
destination.shutdown(socket.SHUT_WR)
|
||||
@@ -54,6 +54,7 @@ from k1link.observatory.source_admission import (
|
||||
from k1link.observatory.worker_agent import (
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerTransport,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
@@ -205,14 +206,19 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorIdentity, ...],
|
||||
) -> Mapping[str, object] | None:
|
||||
self._require_claimant(claimant_id)
|
||||
payload = self._json_request(
|
||||
"POST",
|
||||
"/api/v1/worker/observatory/recorded-jobs/claims",
|
||||
json_body={
|
||||
"schema_version": "missioncore.observatory-worker-claim-request/v1",
|
||||
"schema_version": "missioncore.observatory-worker-claim-request/v2",
|
||||
"claim_request_id": claim_request_id,
|
||||
"supported_executor_identities": [
|
||||
identity.as_dict()
|
||||
for identity in sorted(supported_executor_identities)
|
||||
],
|
||||
},
|
||||
allow_empty=True,
|
||||
)
|
||||
|
||||
@@ -21,6 +21,11 @@ from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
InstalledLabPackage,
|
||||
InstalledLabPackageError,
|
||||
InstalledLabPackageRegistry,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
@@ -77,6 +82,15 @@ class ObservatoryWorkerExecutorBuilder(Protocol):
|
||||
) -> ObservatoryWorkerExecutorRegistration: ...
|
||||
|
||||
|
||||
class ObservatoryWorkerPackageExecutorFactory(Protocol):
|
||||
"""One generic launcher factory shared by every conforming LAB package."""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
context: ObservatoryWorkerPackageExecutorBuildContext,
|
||||
) -> ObservatoryWorkerExecutorRegistration: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorBuildContext:
|
||||
"""Fixed local inputs shared with one install-time executor builder."""
|
||||
@@ -91,6 +105,17 @@ class ObservatoryWorkerExecutorBuildContext:
|
||||
_absolute_path(self.work_root, "Worker build work root")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerPackageExecutorBuildContext(ObservatoryWorkerExecutorBuildContext):
|
||||
"""Generic package plus the same fixed Worker-owned transport boundary."""
|
||||
|
||||
package: InstalledLabPackage
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
ObservatoryWorkerExecutorBuildContext.__post_init__(self)
|
||||
self.package.bind(self.definition, self.candidate)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorBuilderRegistration:
|
||||
"""Local setup-to-builder binding; queued jobs cannot populate this map."""
|
||||
@@ -299,6 +324,107 @@ def compose_installed_observatory_worker_service_from_builders(
|
||||
del bearer_token
|
||||
|
||||
|
||||
def compose_installed_observatory_worker_service_from_packages(
|
||||
*,
|
||||
configuration: ObservatoryWorkerServiceConfiguration,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
runtime_registry: PortableWorkerRuntimeRegistry,
|
||||
packages: InstalledLabPackageRegistry,
|
||||
executor_factory: ObservatoryWorkerPackageExecutorFactory,
|
||||
http_transport: httpx.BaseTransport | None = None,
|
||||
) -> InstalledObservatoryWorkerService:
|
||||
"""Compose all ready LABs through one package-aware executor factory."""
|
||||
|
||||
bearer_token = load_observatory_worker_bearer_token(configuration.bearer_token_file)
|
||||
gateway: ObservatoryWorkerHttpGateway | None = None
|
||||
try:
|
||||
gateway = ObservatoryWorkerHttpGateway(
|
||||
base_url=configuration.base_url,
|
||||
bearer_token=bearer_token,
|
||||
work_root=configuration.work_root,
|
||||
transport=http_transport,
|
||||
)
|
||||
executors = build_ready_executor_registry_from_packages(
|
||||
definitions=definitions,
|
||||
runtime_registry=runtime_registry,
|
||||
packages=packages,
|
||||
executor_factory=executor_factory,
|
||||
source_transport=gateway,
|
||||
result_transport=gateway,
|
||||
work_root=configuration.work_root,
|
||||
)
|
||||
return InstalledObservatoryWorkerService(
|
||||
configuration=configuration,
|
||||
gateway=gateway,
|
||||
agent=ObservatoryWorkerAgent(transport=gateway, executors=executors),
|
||||
)
|
||||
except Exception:
|
||||
if gateway is not None:
|
||||
gateway.close()
|
||||
raise
|
||||
finally:
|
||||
del bearer_token
|
||||
|
||||
|
||||
def build_ready_executor_registry_from_packages(
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
runtime_registry: PortableWorkerRuntimeRegistry,
|
||||
packages: InstalledLabPackageRegistry,
|
||||
executor_factory: ObservatoryWorkerPackageExecutorFactory,
|
||||
source_transport: PortableWorkerSourceMaterializer,
|
||||
result_transport: PortableWorkerResultPublisher,
|
||||
work_root: Path,
|
||||
) -> ObservatoryWorkerExecutorRegistry:
|
||||
"""Build this Worker's installed ready subset with one generic factory."""
|
||||
|
||||
_absolute_path(work_root, "Worker package build work root")
|
||||
ready = definitions.ready_recorded_definitions()
|
||||
ready_keys = {(item.setup_id, item.definition_sha256) for item in ready}
|
||||
package_keys = {
|
||||
(package.setup_id, package.definition_sha256) for package in packages.packages
|
||||
}
|
||||
if not package_keys.issubset(ready_keys):
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"installed LAB packages must bind only ready RunDefinitions"
|
||||
)
|
||||
registrations: list[ObservatoryWorkerExecutorRegistration] = []
|
||||
for package in sorted(
|
||||
packages.packages,
|
||||
key=lambda item: (item.setup_id, item.definition_sha256),
|
||||
):
|
||||
definition = definitions.resolve(
|
||||
package.setup_id,
|
||||
package.definition_sha256,
|
||||
)
|
||||
candidate = runtime_registry.resolve(
|
||||
definition.setup_id,
|
||||
definition.definition_sha256,
|
||||
)
|
||||
try:
|
||||
package.bind(definition, candidate)
|
||||
except InstalledLabPackageError as exc:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"installed LAB package is not bound to its ready runtime"
|
||||
) from exc
|
||||
built = executor_factory(
|
||||
ObservatoryWorkerPackageExecutorBuildContext(
|
||||
definition=definition,
|
||||
candidate=candidate,
|
||||
source_transport=source_transport,
|
||||
result_transport=result_transport,
|
||||
work_root=work_root,
|
||||
package=package,
|
||||
)
|
||||
)
|
||||
if built.identity != package.executor_identity:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"generic package factory returned another executor identity"
|
||||
)
|
||||
registrations.append(built)
|
||||
return ObservatoryWorkerExecutorRegistry(tuple(registrations))
|
||||
|
||||
|
||||
def build_ready_executor_registry(
|
||||
*,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
|
||||
@@ -47,6 +47,9 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_publication_reconciler import (
|
||||
PortablePublicationReconciler,
|
||||
)
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingError,
|
||||
PortableRecordedQueueBindingService,
|
||||
@@ -58,6 +61,7 @@ from k1link.observatory.portable_result_contract import (
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
resolve_published_portable_calculation_profile,
|
||||
)
|
||||
from k1link.observatory.portable_result_view import PortableResultViewService
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
@@ -478,6 +482,18 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||
# Failure remains isolated from K1, Simulation and legacy LAB.
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = None
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
||||
OBSERVATORY_PUBLICATION_RECONCILER = (
|
||||
None
|
||||
if (
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE is None
|
||||
or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||
)
|
||||
else PortablePublicationReconciler(
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport,
|
||||
result_publisher=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher,
|
||||
)
|
||||
)
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED
|
||||
@@ -539,6 +555,7 @@ try:
|
||||
registry=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
capability_probe=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||
dispatch_available=OBSERVATORY_WORKER_DISPATCH_READY,
|
||||
equipment_capture_registry=session_store.equipment_capture_registry,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||
except (
|
||||
@@ -825,9 +842,22 @@ async def _recording_preparation_reconciler() -> None:
|
||||
await asyncio.sleep(2.0)
|
||||
|
||||
|
||||
async def _portable_result_publication_reconciler() -> None:
|
||||
service = OBSERVATORY_PUBLICATION_RECONCILER
|
||||
if service is None:
|
||||
return
|
||||
while True:
|
||||
# Durable state remains pending/failed and is retried on the next
|
||||
# bounded pass or through the explicit operator action.
|
||||
with suppress(OSError, ValueError):
|
||||
await asyncio.to_thread(service.run_once)
|
||||
await asyncio.sleep(15.0)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler: asyncio.Task[None] | None = None
|
||||
publication_reconciler: asyncio.Task[None] | None = None
|
||||
try:
|
||||
configure_scanner_diagnostics(session_store.data_dir / "logs")
|
||||
session_recording_preparation_manager.start()
|
||||
@@ -841,6 +871,9 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
# expensive on field captures. Start it immediately in the background
|
||||
# instead of holding the ASGI startup gate.
|
||||
reconciler = asyncio.create_task(_recording_preparation_reconciler())
|
||||
publication_reconciler = asyncio.create_task(
|
||||
_portable_result_publication_reconciler()
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
await map_gateway_proxy.close()
|
||||
@@ -848,6 +881,10 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await reconciler
|
||||
if publication_reconciler is not None:
|
||||
publication_reconciler.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await publication_reconciler
|
||||
await asyncio.to_thread(session_recording_preparation_manager.close)
|
||||
await asyncio.to_thread(lidar_local_surface_read_service.close)
|
||||
plugin_environment.close()
|
||||
@@ -1036,6 +1073,24 @@ app.include_router(
|
||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||
portable_binding_service=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||
portable_result_view=(
|
||||
None
|
||||
if session_artifact_gateway is None
|
||||
else PortableResultViewService(
|
||||
sessions=session_store,
|
||||
artifacts=session_artifact_gateway.store,
|
||||
)
|
||||
),
|
||||
portable_artifact_transport=(
|
||||
None
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||
else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
|
||||
),
|
||||
portable_result_publisher=(
|
||||
None
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||
else OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
|
||||
),
|
||||
)
|
||||
)
|
||||
if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory import (
|
||||
@@ -22,12 +23,24 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
)
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingError,
|
||||
PortableQueueBindingIntegrityError,
|
||||
PortableQueueBindingStaleCheckError,
|
||||
PortableRecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import PortableResultPublisherError
|
||||
from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.portable_result_view import (
|
||||
PortableResultViewError,
|
||||
PortableResultViewService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionUnavailableError,
|
||||
)
|
||||
@@ -158,10 +171,32 @@ def build_observatory_router(
|
||||
portable_setup_projector: PortableSetupProjector | PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector_error: str | None = None,
|
||||
portable_binding_service: PortableRecordedQueueBindingService | None = None,
|
||||
portable_result_view: PortableResultViewService | None = None,
|
||||
portable_artifact_transport: PortableObservatoryArtifactTransport | None = None,
|
||||
portable_result_publisher: PortableObservatoryResultPublisher | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
router = APIRouter(tags=["observatory"])
|
||||
if (portable_artifact_transport is None) != (portable_result_publisher is None):
|
||||
raise ValueError("portable publication dependencies must be configured together")
|
||||
|
||||
if portable_result_view is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-results/{result_id}")
|
||||
def get_portable_result_view(
|
||||
result_id: Annotated[
|
||||
str,
|
||||
ApiPath(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"),
|
||||
],
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
return portable_result_view.read(result_id)
|
||||
except PortableResultViewError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Portable-результат недоступен для универсального просмотра.",
|
||||
) from exc
|
||||
|
||||
def source_summary(session_id: str) -> SessionSummary:
|
||||
try:
|
||||
@@ -1092,6 +1127,80 @@ def build_observatory_router(
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
|
||||
if (
|
||||
portable_artifact_transport is not None
|
||||
and portable_result_publisher is not None
|
||||
):
|
||||
|
||||
@router.post(
|
||||
"/api/v1/observatory/runs/{job_id}/publication/retry",
|
||||
response_model=None,
|
||||
)
|
||||
def retry_observatory_result_publication(
|
||||
job_id: str = ApiPath(
|
||||
min_length=48,
|
||||
max_length=48,
|
||||
pattern=r"^observatory-run-[a-f0-9]{32}$",
|
||||
),
|
||||
) -> dict[str, object] | JSONResponse:
|
||||
"""Retry verification/publication only; never repeat compute."""
|
||||
|
||||
try:
|
||||
job = recorded_job_queue.get(job_id)
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Расчёт Обсерватории не найден.",
|
||||
) from exc
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Durable-очередь расчётов недоступна.",
|
||||
) from exc
|
||||
if job.publication_state == "published":
|
||||
return job.as_dict()
|
||||
if job.state != "succeeded" or job.publication_state not in {
|
||||
"pending",
|
||||
"failed",
|
||||
}:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Результат не ожидает повторной публикации.",
|
||||
)
|
||||
try:
|
||||
package_root = portable_artifact_transport.package_root_for_terminal(
|
||||
job
|
||||
)
|
||||
portable_result_publisher.publish(
|
||||
job=job,
|
||||
package_root=package_root,
|
||||
)
|
||||
except PortableResultPublisherError as exc:
|
||||
message = (" ".join(str(exc).split()) or "Publication failed.")[:1_000]
|
||||
try:
|
||||
failed = recorded_job_queue.mark_publication_failed(
|
||||
job_id,
|
||||
message=message,
|
||||
)
|
||||
except (ObservatoryRecordedQueueError, ValueError) as queue_exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Состояние публикации не удалось сохранить.",
|
||||
) from queue_exc
|
||||
return JSONResponse(status_code=202, content=failed.as_dict())
|
||||
except PortableArtifactTransportError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Пакет результата недоступен для повторной публикации.",
|
||||
) from exc
|
||||
try:
|
||||
return recorded_job_queue.mark_published(job_id).as_dict()
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Состояние публикации не удалось сохранить.",
|
||||
) from exc
|
||||
|
||||
elif recorded_job_queue_error is not None:
|
||||
|
||||
@router.post("/api/v1/observatory/runs")
|
||||
|
||||
@@ -20,9 +20,9 @@ from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
MAX_RESULT_MANIFEST_BYTES,
|
||||
@@ -38,6 +38,7 @@ from k1link.observatory.portable_result_publisher import (
|
||||
PortableObservatoryResultPublisher,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
MAX_RECORDED_EXECUTOR_CAPABILITIES,
|
||||
ObservatoryRecordedCheckpointError,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedPreemptionError,
|
||||
@@ -48,9 +49,12 @@ from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedQueueIntegrityError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
RecordedExecutorIdentity,
|
||||
)
|
||||
|
||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||
OBSERVATORY_WORKER_CAPABILITY_CLAIM_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-claim-request/v2"
|
||||
)
|
||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||
OBSERVATORY_WORKER_RENEW_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-renew-request/v1"
|
||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||
@@ -143,13 +147,36 @@ class _StrictWorkerRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutorCapability(_StrictWorkerRequest):
|
||||
release_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
image_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
model_manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
resource_profile_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
def recorded_identity(self) -> RecordedExecutorIdentity:
|
||||
return RecordedExecutorIdentity(**self.model_dump())
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-claim-request/v1"]
|
||||
schema_version: Literal["missioncore.observatory-worker-claim-request/v2"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
supported_executor_identities: tuple[ObservatoryWorkerExecutorCapability, ...] = Field(
|
||||
max_length=MAX_RECORDED_EXECUTOR_CAPABILITIES,
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_capability_snapshot(self) -> ObservatoryWorkerClaimRequest:
|
||||
identities = tuple(
|
||||
capability.recorded_identity()
|
||||
for capability in self.supported_executor_identities
|
||||
)
|
||||
if len(identities) != len(set(identities)):
|
||||
raise ValueError("executor capabilities must be unique")
|
||||
return self
|
||||
|
||||
|
||||
class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
||||
@@ -257,6 +284,10 @@ def build_observatory_worker_router(
|
||||
lambda: queue.claim_next(
|
||||
claimant_id=authentication.contour_id,
|
||||
claim_request_id=request.claim_request_id,
|
||||
supported_executor_identities=tuple(
|
||||
capability.recorded_identity()
|
||||
for capability in request.supported_executor_identities
|
||||
),
|
||||
)
|
||||
)
|
||||
if claim is None:
|
||||
@@ -303,11 +334,11 @@ def build_observatory_worker_router(
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/succeed")
|
||||
@router.post("/recorded-jobs/{job_id}/succeed", response_model=None)
|
||||
def succeed_job(
|
||||
request: ObservatoryWorkerSucceedRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
) -> dict[str, object] | JSONResponse:
|
||||
if artifact_transport is not None:
|
||||
_artifact_call(
|
||||
lambda: artifact_transport.require_completed_for_success(
|
||||
@@ -318,29 +349,76 @@ def build_observatory_worker_router(
|
||||
claimant_id=authentication.contour_id,
|
||||
)
|
||||
)
|
||||
succeeded = _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
)
|
||||
if artifact_transport is not None and result_publisher is not None:
|
||||
succeeded = _queue_call(
|
||||
lambda: queue.complete_for_publication(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
)
|
||||
if succeeded.publication_state == "published":
|
||||
return succeeded.as_dict()
|
||||
package_root = _artifact_call(
|
||||
lambda: artifact_transport.package_root_for_terminal(succeeded)
|
||||
)
|
||||
try:
|
||||
result_publisher.publish(job=succeeded, package_root=package_root)
|
||||
except PortableResultPublisherError as exc:
|
||||
publication_error = _publication_error(exc)
|
||||
failed = _queue_call(
|
||||
lambda: queue.mark_publication_failed(
|
||||
job_id,
|
||||
message=publication_error,
|
||||
)
|
||||
)
|
||||
return JSONResponse(status_code=202, content=failed.as_dict())
|
||||
return _queue_call(lambda: queue.mark_published(job_id)).as_dict()
|
||||
return _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
if artifact_transport is not None and result_publisher is not None:
|
||||
|
||||
@router.post(
|
||||
"/recorded-jobs/{job_id}/publication/retry",
|
||||
response_model=None,
|
||||
)
|
||||
def retry_publication(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object] | JSONResponse:
|
||||
job = _queue_call(lambda: queue.get(job_id))
|
||||
if job.publication_state == "published":
|
||||
return job.as_dict()
|
||||
if job.state != "succeeded" or job.publication_state not in {
|
||||
"pending",
|
||||
"failed",
|
||||
}:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail=(
|
||||
"Recorded result is sealed but its verified publication "
|
||||
"requires reconciliation."
|
||||
),
|
||||
) from exc
|
||||
return succeeded.as_dict()
|
||||
status_code=409,
|
||||
detail="Recorded result is not awaiting publication.",
|
||||
)
|
||||
package_root = _artifact_call(
|
||||
lambda: artifact_transport.package_root_for_terminal(job)
|
||||
)
|
||||
try:
|
||||
result_publisher.publish(job=job, package_root=package_root)
|
||||
except PortableResultPublisherError as exc:
|
||||
publication_error = _publication_error(exc)
|
||||
failed = _queue_call(
|
||||
lambda: queue.mark_publication_failed(
|
||||
job_id,
|
||||
message=publication_error,
|
||||
)
|
||||
)
|
||||
return JSONResponse(status_code=202, content=failed.as_dict())
|
||||
return _queue_call(lambda: queue.mark_published(job_id)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/fail")
|
||||
def fail_job(
|
||||
@@ -650,6 +728,11 @@ def _raise(exc: Exception) -> None:
|
||||
raise exc
|
||||
|
||||
|
||||
def _publication_error(exc: PortableResultPublisherError) -> str:
|
||||
message = " ".join(str(exc).split())
|
||||
return (message or "Portable result publication failed.")[:1_000]
|
||||
|
||||
|
||||
async def _read_bounded_body(request: Request, maximum_bytes: int) -> bytes:
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length is not None:
|
||||
|
||||
Reference in New Issue
Block a user