feat(observatory): add portable LAB V1 foundation
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
"""Portable recorded-source admission bound to the durable Observatory queue.
|
||||
|
||||
The binding resolves a server-owned portable RunDefinition before inspecting a
|
||||
session. It therefore cannot write source contracts or jobs for an unavailable
|
||||
executor. A successful check is fenced by a content digest; admission repeats
|
||||
the source read and persists only if the exact checked snapshot still holds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobIntent,
|
||||
ObservatoryRecordedJobQueue,
|
||||
RecordedRunDefinition,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceAdmission,
|
||||
PortableRecordedSourceCapability,
|
||||
PortableSourceAdmissionStaleError,
|
||||
RecordedK1SourceAdmissionService,
|
||||
)
|
||||
from k1link.sessions.media import RecordedMediaInspector
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
PORTABLE_QUEUE_BINDING_SCHEMA: Final = "missioncore.observatory-portable-queue-binding/v1"
|
||||
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDEMPOTENCY_KEY: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$")
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class PortableQueueBindingError(RuntimeError):
|
||||
"""A portable source cannot be bound to the recorded queue."""
|
||||
|
||||
|
||||
class PortableQueueBindingIntegrityError(PortableQueueBindingError):
|
||||
"""A registry, source admission, or queue identity disagrees."""
|
||||
|
||||
|
||||
class PortableQueueBindingStaleCheckError(PortableQueueBindingError):
|
||||
"""The source changed after the operator-visible compatibility check."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedRunPreparation:
|
||||
"""Path-free immutable bridge from source admission to a queue intent."""
|
||||
|
||||
definition: RecordedRunDefinition
|
||||
source: PortableRecordedSourceAdmission
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.definition.source_adapter_sha256 != self.source.source_adapter_sha256:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and source admission adapter identities disagree"
|
||||
)
|
||||
|
||||
@property
|
||||
def check_sha256(self) -> str:
|
||||
return _sha256(self.identity_document())
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_QUEUE_BINDING_SCHEMA,
|
||||
"source": self.source.as_dict(),
|
||||
"definition": _definition_document(self.definition),
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
def intent(self, *, idempotency_key: str) -> ObservatoryRecordedJobIntent:
|
||||
_pattern(idempotency_key, _IDEMPOTENCY_KEY, "idempotency key")
|
||||
return ObservatoryRecordedJobIntent(
|
||||
idempotency_key=idempotency_key,
|
||||
source_session_id=self.source.source_session_id,
|
||||
source_catalog_sha256=self.source.source_catalog_sha256,
|
||||
source_bundle_sha256=self.source.source_bundle_sha256,
|
||||
source_capability_manifest_sha256=(self.source.source_capability_manifest_sha256),
|
||||
setup_id=self.definition.setup_id,
|
||||
definition_sha256=self.definition.definition_sha256,
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {**self.identity_document(), "check_sha256": self.check_sha256}
|
||||
|
||||
|
||||
class PortableRecordedQueueBindingService:
|
||||
"""Resolve, check, admit, and submit portable recorded computations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
data_dir: Path,
|
||||
session_store: SessionStore,
|
||||
media_inspector: RecordedMediaInspector,
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
queue: ObservatoryRecordedJobQueue | None = None,
|
||||
) -> None:
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
if self.data_dir != session_store.data_dir:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable binding and SessionStore roots disagree"
|
||||
)
|
||||
if queue is not None and self.data_dir != queue.data_dir:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable binding and recorded queue roots disagree"
|
||||
)
|
||||
self._session_store = session_store
|
||||
self._media_inspector = media_inspector
|
||||
self._definitions = definitions
|
||||
self._queue = queue
|
||||
|
||||
def probe(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedSourceCapability:
|
||||
"""Return cheap authoritative compatibility without replay preparation."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
capability = self._source_service(portable).probe(source_session_id)
|
||||
if recorded.source_adapter_sha256 != capability.source_adapter_sha256:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and source capability adapter identities disagree"
|
||||
)
|
||||
return capability
|
||||
|
||||
def check(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
"""Verify one source/setup combination without persistent writes."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
source_service = self._source_service(portable)
|
||||
return self._bind(recorded, source_service.check(source_session_id))
|
||||
|
||||
def admit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
expected_check_sha256: str,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
"""Persist exact checked contracts and reject changed source state."""
|
||||
|
||||
_digest(expected_check_sha256, "expected portable check sha256")
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
source_service = self._source_service(portable)
|
||||
checked = self._bind(recorded, source_service.check(source_session_id))
|
||||
if checked.check_sha256 != expected_check_sha256:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source or definition changed after its check"
|
||||
)
|
||||
try:
|
||||
source = source_service.admit(
|
||||
source_session_id,
|
||||
expected_admission_sha256=checked.source.identity_sha256,
|
||||
)
|
||||
except PortableSourceAdmissionStaleError as exc:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source changed while admission was being committed"
|
||||
) from exc
|
||||
admitted = self._bind(recorded, source)
|
||||
if admitted.check_sha256 != expected_check_sha256:
|
||||
raise PortableQueueBindingStaleCheckError(
|
||||
"portable source changed while admission was being committed"
|
||||
)
|
||||
return admitted
|
||||
|
||||
def submit(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
expected_check_sha256: str,
|
||||
idempotency_key: str,
|
||||
enqueue: bool = True,
|
||||
) -> tuple[ObservatoryRecordedJob, bool]:
|
||||
"""Admit and atomically submit the resulting path-free queue intent."""
|
||||
|
||||
if self._queue is None:
|
||||
raise PortableQueueBindingIntegrityError("portable recorded queue is unavailable")
|
||||
preparation = self.admit(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition_sha256,
|
||||
expected_check_sha256=expected_check_sha256,
|
||||
)
|
||||
return self._queue.submit(
|
||||
preparation.intent(idempotency_key=idempotency_key),
|
||||
enqueue=enqueue,
|
||||
)
|
||||
|
||||
def _resolve_definition(
|
||||
self,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> tuple[PortableRunDefinition, RecordedRunDefinition]:
|
||||
portable = self._definitions.resolve(setup_id, definition_sha256)
|
||||
# This conversion is deliberately first: not-installed executors fail
|
||||
# before the SessionStore, immutable document store, or queue is touched.
|
||||
recorded = portable.to_recorded_run_definition()
|
||||
if self._queue is not None:
|
||||
try:
|
||||
queue_definition = self._queue.resolve_definition(
|
||||
setup_id,
|
||||
definition_sha256,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable definition is absent from the recorded queue"
|
||||
) from exc
|
||||
if queue_definition != recorded:
|
||||
raise PortableQueueBindingIntegrityError(
|
||||
"portable registry and recorded queue definitions disagree"
|
||||
)
|
||||
return portable, recorded
|
||||
|
||||
def _source_service(
|
||||
self,
|
||||
definition: PortableRunDefinition,
|
||||
) -> RecordedK1SourceAdmissionService:
|
||||
return RecordedK1SourceAdmissionService(
|
||||
data_dir=self.data_dir,
|
||||
session_store=self._session_store,
|
||||
media_inspector=self._media_inspector,
|
||||
requirements=definition.to_source_admission_requirements(),
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _bind(
|
||||
definition: RecordedRunDefinition,
|
||||
source: PortableRecordedSourceAdmission,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
return PortableRecordedRunPreparation(definition=definition, source=source)
|
||||
|
||||
|
||||
def _definition_document(definition: RecordedRunDefinition) -> dict[str, object]:
|
||||
return {
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_id": definition.definition_id,
|
||||
"definition_version": definition.definition_version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"source_adapter_id": definition.source_adapter_id,
|
||||
"source_adapter_version": definition.source_adapter_version,
|
||||
"source_adapter_sha256": definition.source_adapter_sha256,
|
||||
"executor_release_id": definition.executor_release_id,
|
||||
"executor_release_sha256": definition.executor_release_sha256,
|
||||
"executor_image_sha256": definition.executor_image_sha256,
|
||||
"model_release_ids": list(definition.model_release_ids),
|
||||
"model_manifest_sha256": definition.model_manifest_sha256,
|
||||
"resource_profile_id": definition.resource_profile_id,
|
||||
"resource_profile_sha256": definition.resource_profile_sha256,
|
||||
"checkpoint_policy": definition.checkpoint_policy,
|
||||
"allowed_checkpoints": list(definition.allowed_checkpoints),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
return _pattern(value, _SHA256, label)
|
||||
|
||||
|
||||
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,234 @@
|
||||
"""UI-ready Observatory projection for the portable LAB V1 definition.
|
||||
|
||||
This module deliberately does not extend the legacy setup registry and does
|
||||
not submit work. It projects two independent facts for one selected source:
|
||||
|
||||
* the result of a lightweight recorded-source capability probe;
|
||||
* the executor state sealed by the portable RunDefinition;
|
||||
|
||||
Keeping those facts separate prevents a compatible new recording from being
|
||||
described as incompatible merely because the executor is not installed yet.
|
||||
Historical vegetation-shadow results belong only to the legacy catalog and
|
||||
never become exact results of the generic portable v2 definition.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
)
|
||||
from k1link.observatory.source_admission import (
|
||||
PortableRecordedSourceCapability,
|
||||
PortableSourceAdmissionError,
|
||||
)
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-setup-catalog/v2"
|
||||
)
|
||||
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||
|
||||
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
_MODEL_PRESENTATION: Final = {
|
||||
"eomt-cityscapes-large-1024-v1": "EoMT Cityscapes Large 1024",
|
||||
"lab-v1-ddrnet-39-goose-fine-64-v1": "DDRNet-39",
|
||||
}
|
||||
|
||||
|
||||
class PortableSetupProjectionError(RuntimeError):
|
||||
"""The portable setup cannot be projected without weakening its contract."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PortableSourceCapabilityProbeService(Protocol):
|
||||
"""A definition-bound lightweight source-capability service."""
|
||||
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
"""Probe one source without preparing media or persisting documents."""
|
||||
|
||||
|
||||
type PortableSourceCapabilityProbe = (
|
||||
Callable[[str], PortableRecordedSourceCapability] | PortableSourceCapabilityProbeService
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SourceCompatibility:
|
||||
compatible: bool
|
||||
capability: PortableRecordedSourceCapability | None
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"outcome": "pass" if self.compatible else "blocked",
|
||||
"compatible": self.compatible,
|
||||
"reason": (
|
||||
"Запись соответствует требованиям EoMT + DDRNet."
|
||||
if self.compatible
|
||||
else "Запись не соответствует требованиям EoMT + DDRNet."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class PortableLabV1SetupProjector:
|
||||
"""Project the generic portable LAB V1 setup for one selected source."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
capability_probe: PortableSourceCapabilityProbe,
|
||||
) -> None:
|
||||
self._definition = _resolve_lab_v1_definition(registry)
|
||||
if not isinstance(
|
||||
capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
) and not callable(capability_probe):
|
||||
raise PortableSetupProjectionError("portable source capability probe is unavailable")
|
||||
self._capability_probe = capability_probe
|
||||
_validate_model_presentation(self._definition)
|
||||
if self._definition.authority.as_dict() != _OBSERVATION_ONLY_AUTHORITY:
|
||||
raise PortableSetupProjectionError("portable LAB V1 authority is not observation-only")
|
||||
|
||||
def catalog(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a one-setup v2 catalog projection for ``source``."""
|
||||
|
||||
return {
|
||||
"schema_version": PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA,
|
||||
"source_session_id": source.session_id,
|
||||
"setups": [self.project(source)],
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def project(self, source: SessionSummary) -> dict[str, object]:
|
||||
"""Return a strict, observation-only setup projection."""
|
||||
|
||||
_validate_source_id(source.session_id)
|
||||
compatibility = self._probe_source(source.session_id)
|
||||
definition = self._definition
|
||||
executor = definition.executor
|
||||
return {
|
||||
"setup_id": definition.setup_id,
|
||||
"display_name": PORTABLE_LAB_V1_DISPLAY_NAME,
|
||||
"description": (
|
||||
"Проверка записанной K1-сессии моделями EoMT и DDRNet; только наблюдение."
|
||||
),
|
||||
"origin": "portable-definition",
|
||||
"source_requirements": definition.source_requirements.as_dict(),
|
||||
"run_definition": {
|
||||
"definition_id": definition.definition_id,
|
||||
"version": definition.version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"result_schema": definition.result_contract.result_schema,
|
||||
"result_kind": definition.result_contract.result_kind,
|
||||
"models": _project_models(definition),
|
||||
},
|
||||
"source_compatibility": compatibility.as_dict(),
|
||||
"executor": {
|
||||
"contour_id": executor.contour_id,
|
||||
"state": executor.state,
|
||||
"ready": executor.ready,
|
||||
"reason": executor.reason,
|
||||
},
|
||||
"existing_results": [],
|
||||
"preflight": {
|
||||
"outcome": "blocked",
|
||||
"action": "blocked",
|
||||
"reason": self._preflight_reason(compatibility),
|
||||
"submission_allowed": False,
|
||||
"existing_result_ids": [],
|
||||
},
|
||||
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
def _probe_source(self, source_session_id: str) -> _SourceCompatibility:
|
||||
try:
|
||||
if isinstance(
|
||||
self._capability_probe,
|
||||
PortableSourceCapabilityProbeService,
|
||||
):
|
||||
capability = self._capability_probe.probe(source_session_id)
|
||||
else:
|
||||
capability = self._capability_probe(source_session_id)
|
||||
except PortableSourceAdmissionError:
|
||||
return _SourceCompatibility(compatible=False, capability=None)
|
||||
if not isinstance(capability, PortableRecordedSourceCapability):
|
||||
raise PortableSetupProjectionError("capability probe returned an invalid result")
|
||||
if capability.source_session_id != source_session_id:
|
||||
raise PortableSetupProjectionError(
|
||||
"capability probe is bound to another source session"
|
||||
)
|
||||
if capability.source_adapter_sha256 != self._definition.source_adapter.contract_sha256:
|
||||
raise PortableSetupProjectionError("capability probe uses another source adapter")
|
||||
return _SourceCompatibility(compatible=True, capability=capability)
|
||||
|
||||
def _preflight_reason(self, compatibility: _SourceCompatibility) -> str:
|
||||
if not compatibility.compatible:
|
||||
return "Запись не соответствует требованиям этого сетапа."
|
||||
if not self._definition.executor.ready:
|
||||
return "Вычислительный контур LAB V1 пока недоступен."
|
||||
return (
|
||||
"Server-side проверка definition/check SHA и постановка portable "
|
||||
"LAB V1 в очередь пока недоступны."
|
||||
)
|
||||
|
||||
|
||||
def _resolve_lab_v1_definition(
|
||||
registry: PortableRunDefinitionRegistry,
|
||||
) -> PortableRunDefinition:
|
||||
matching = tuple(
|
||||
definition
|
||||
for definition in registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
if len(matching) != 1:
|
||||
raise PortableSetupProjectionError("portable LAB V1 definition is unavailable or ambiguous")
|
||||
return matching[0]
|
||||
|
||||
|
||||
def _validate_model_presentation(definition: PortableRunDefinition) -> None:
|
||||
releases = {model.release_id: model for model in definition.models}
|
||||
if set(releases) != set(_MODEL_PRESENTATION):
|
||||
raise PortableSetupProjectionError(
|
||||
"portable LAB V1 model set does not match its presentation contract"
|
||||
)
|
||||
eomt = releases["eomt-cityscapes-large-1024-v1"]
|
||||
ddrnet = releases["lab-v1-ddrnet-39-goose-fine-64-v1"]
|
||||
if (
|
||||
eomt.model_id != "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||
or ddrnet.architecture != "ddrnet_39"
|
||||
):
|
||||
raise PortableSetupProjectionError(
|
||||
"portable LAB V1 model identities do not match their presentation"
|
||||
)
|
||||
|
||||
|
||||
def _project_models(definition: PortableRunDefinition) -> list[dict[str, object]]:
|
||||
by_release = {model.release_id: model for model in definition.models}
|
||||
return [
|
||||
{
|
||||
"name": _MODEL_PRESENTATION[release_id],
|
||||
"release_id": release_id,
|
||||
"model_id": by_release[release_id].model_id,
|
||||
"architecture": by_release[release_id].architecture,
|
||||
}
|
||||
for release_id in _MODEL_PRESENTATION
|
||||
]
|
||||
|
||||
|
||||
def _validate_source_id(source_session_id: str) -> None:
|
||||
if _SOURCE_ID.fullmatch(source_session_id) is None:
|
||||
raise PortableSetupProjectionError("source session id is invalid")
|
||||
@@ -34,16 +34,10 @@ from uuid import uuid4
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA: Final = "missioncore.observatory-recorded-job/v1"
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-recorded-job-request/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA: Final = (
|
||||
"missioncore.observatory-recorded-job-claim/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA: Final = "missioncore.observatory-recorded-job-request/v1"
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA: Final = "missioncore.observatory-recorded-job-claim/v1"
|
||||
OBSERVATORY_LIVE_LEASE_SCHEMA: Final = "missioncore.observatory-live-k1-lease/v1"
|
||||
OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-live-k1-lease-request/v1"
|
||||
)
|
||||
OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA: Final = "missioncore.observatory-live-k1-lease-request/v1"
|
||||
RECORDED_JOB_DATABASE_NAME: Final = "observatory-recorded-jobs.sqlite3"
|
||||
MAX_RECORDED_JOBS: Final = 10_000
|
||||
MAX_RECORDED_CLAIM_RECEIPTS: Final = 50_000
|
||||
@@ -67,9 +61,7 @@ type RecordedJobState = Literal[
|
||||
"reconciliation-required",
|
||||
]
|
||||
type CheckpointPolicy = Literal["cooperative", "non-checkpointable"]
|
||||
type LiveLeaseState = Literal[
|
||||
"pending", "active", "completed", "failed", "cancelled"
|
||||
]
|
||||
type LiveLeaseState = Literal["pending", "active", "completed", "failed", "cancelled"]
|
||||
type LiveTerminalOutcome = Literal["completed", "failed", "cancelled"]
|
||||
|
||||
_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||
@@ -80,9 +72,7 @@ _SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_CHECKPOINT_ID = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_TERMINAL_STATES: Final = frozenset(
|
||||
{"succeeded", "failed", "reconciliation-required"}
|
||||
)
|
||||
_TERMINAL_STATES: Final = frozenset({"succeeded", "failed", "reconciliation-required"})
|
||||
_OPEN_LIVE_STATES: Final = frozenset({"pending", "active"})
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
@@ -299,8 +289,7 @@ class RecordedRunDefinitionRegistry:
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
keys = [
|
||||
(definition.setup_id, definition.definition_sha256)
|
||||
for definition in self.definitions
|
||||
(definition.setup_id, definition.definition_sha256) for definition in self.definitions
|
||||
]
|
||||
version_keys = [
|
||||
(definition.definition_id, definition.definition_version)
|
||||
@@ -358,9 +347,7 @@ class ObservatoryRecordedJobIntent:
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (
|
||||
self.source_capability_manifest_sha256
|
||||
),
|
||||
"source_capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"setup_id": self.setup_id,
|
||||
"definition_sha256": self.definition_sha256,
|
||||
}
|
||||
@@ -471,18 +458,12 @@ class ObservatoryRecordedJob:
|
||||
"recorded-job preemption marker is invalid"
|
||||
)
|
||||
if self.claim_generation < 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim generation is invalid"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim generation is invalid")
|
||||
_validate_optional_pattern(self.active_claim_token, _TOKEN, "active claim token")
|
||||
_validate_optional_pattern(self.active_claimant_id, _IDENTIFIER, "claimant id")
|
||||
_validate_optional_pattern(
|
||||
self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id"
|
||||
)
|
||||
_validate_optional_pattern(self.last_checkpoint_id, _CHECKPOINT_ID, "last checkpoint id")
|
||||
if not isinstance(self.restart_from_zero, bool):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job restart marker is invalid"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job restart marker is invalid")
|
||||
_validate_optional_pattern(
|
||||
self.preemption_receipt_sha256,
|
||||
_SHA256,
|
||||
@@ -513,9 +494,7 @@ class ObservatoryRecordedJob:
|
||||
"session_id": self.source_session_id,
|
||||
"catalog_sha256": self.source_catalog_sha256,
|
||||
"bundle_sha256": self.source_bundle_sha256,
|
||||
"capability_manifest_sha256": (
|
||||
self.source_capability_manifest_sha256
|
||||
),
|
||||
"capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"adapter": {
|
||||
"adapter_id": self.source_adapter_id,
|
||||
"version": self.source_adapter_version,
|
||||
@@ -661,9 +640,7 @@ class ObservatoryLiveLease:
|
||||
and self.terminal_request_sha256 is not None
|
||||
and self.terminated_at_utc is not None
|
||||
):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"live terminal identity is incomplete"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("live terminal identity is incomplete")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
@@ -730,9 +707,7 @@ class ObservatoryNonCheckpointableCancellationRequest:
|
||||
"schema_version": OBSERVATORY_LIVE_LEASE_REQUEST_SCHEMA,
|
||||
"cancellation_request_id": self.cancellation_request_id,
|
||||
"job_id": self.job_id,
|
||||
"claim_token_sha256": hashlib.sha256(
|
||||
self.claim_token.encode()
|
||||
).hexdigest(),
|
||||
"claim_token_sha256": hashlib.sha256(self.claim_token.encode()).hexdigest(),
|
||||
"claim_generation": self.claim_generation,
|
||||
"live_trigger_id": self.live_trigger_id,
|
||||
"executor_release_sha256": self.executor_release_sha256,
|
||||
@@ -781,11 +756,7 @@ class ObservatoryNonCheckpointableCancellationReceipt:
|
||||
_validate_pattern(self.cancellation_id, _IDEMPOTENCY_KEY, "cancellation id")
|
||||
_validate_digest(self.request_sha256, "cancellation request sha256")
|
||||
_validate_digest(self.receipt_sha256, "cancellation receipt sha256")
|
||||
if not (
|
||||
self.resources_released
|
||||
and self.staging_discarded
|
||||
and self.restart_from_zero
|
||||
):
|
||||
if not (self.resources_released and self.staging_discarded and self.restart_from_zero):
|
||||
raise ValueError(
|
||||
"cancellation receipt must release resources, discard staging, "
|
||||
"and require restart from zero"
|
||||
@@ -819,17 +790,13 @@ class ObservatoryRecordedJobQueue:
|
||||
*,
|
||||
definitions: RecordedRunDefinitionRegistry,
|
||||
clock: Callable[[], str] = utc_now_iso,
|
||||
non_checkpointable_preemptor: (
|
||||
ObservatoryNonCheckpointablePreemptor | None
|
||||
) = None,
|
||||
non_checkpointable_preemptor: (ObservatoryNonCheckpointablePreemptor | None) = None,
|
||||
max_jobs: int = MAX_RECORDED_JOBS,
|
||||
max_claim_receipts: int = MAX_RECORDED_CLAIM_RECEIPTS,
|
||||
max_live_leases: int = MAX_LIVE_LEASES,
|
||||
) -> None:
|
||||
_validate_quota(max_jobs, MAX_RECORDED_JOBS, "recorded job")
|
||||
_validate_quota(
|
||||
max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt"
|
||||
)
|
||||
_validate_quota(max_claim_receipts, MAX_RECORDED_CLAIM_RECEIPTS, "claim receipt")
|
||||
_validate_quota(max_live_leases, MAX_LIVE_LEASES, "live lease")
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
self.database_path = self.data_dir / RECORDED_JOB_DATABASE_NAME
|
||||
@@ -842,6 +809,15 @@ class ObservatoryRecordedJobQueue:
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
def resolve_definition(
|
||||
self,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> RecordedRunDefinition:
|
||||
"""Expose the immutable server allowlist without exposing queue internals."""
|
||||
|
||||
return self._definitions.resolve(setup_id, definition_sha256)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
intent: ObservatoryRecordedJobIntent,
|
||||
@@ -875,9 +851,7 @@ class ObservatoryRecordedJobQueue:
|
||||
limit=self._max_jobs,
|
||||
label="recorded job",
|
||||
)
|
||||
definition = self._definitions.resolve(
|
||||
intent.setup_id, intent.definition_sha256
|
||||
)
|
||||
definition = self._definitions.resolve(intent.setup_id, intent.definition_sha256)
|
||||
now = self._timestamp()
|
||||
job_id = f"observatory-run-{uuid4().hex}"
|
||||
identity_sha256 = _job_identity_sha256(intent, definition)
|
||||
@@ -973,8 +947,7 @@ class ObservatoryRecordedJobQueue:
|
||||
request_sha256 = _claim_request_sha256(claimant_id, claim_request_id)
|
||||
with self._transaction() as connection:
|
||||
receipt = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_claim_receipts "
|
||||
"WHERE claim_request_id = ?",
|
||||
"SELECT * FROM observatory_recorded_claim_receipts WHERE claim_request_id = ?",
|
||||
(claim_request_id,),
|
||||
).fetchone()
|
||||
if receipt is not None:
|
||||
@@ -1062,11 +1035,7 @@ class ObservatoryRecordedJobQueue:
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
f"cannot start recorded job from {job.state}"
|
||||
)
|
||||
state = (
|
||||
"paused"
|
||||
if self._open_live_lease_row(connection) is not None
|
||||
else "running"
|
||||
)
|
||||
state = "paused" if self._open_live_lease_row(connection) is not None else "running"
|
||||
connection.execute(
|
||||
"UPDATE observatory_recorded_jobs SET state = ?, "
|
||||
"preemption_requested = ?, updated_at_utc = ? WHERE job_id = ?",
|
||||
@@ -1179,14 +1148,11 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._read_connection() as connection:
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def get_by_idempotency_key(
|
||||
self, idempotency_key: str
|
||||
) -> ObservatoryRecordedJob:
|
||||
def get_by_idempotency_key(self, idempotency_key: str) -> ObservatoryRecordedJob:
|
||||
_validate_pattern(idempotency_key, _IDEMPOTENCY_KEY, "idempotency key")
|
||||
with self._read_connection() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs "
|
||||
"WHERE idempotency_key = ?",
|
||||
"SELECT * FROM observatory_recorded_jobs WHERE idempotency_key = ?",
|
||||
(idempotency_key,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
@@ -1226,9 +1192,7 @@ class ObservatoryRecordedJobQueue:
|
||||
).fetchall()
|
||||
return tuple(_job_from_row(row) for row in rows)
|
||||
|
||||
def request_live(
|
||||
self, intent: ObservatoryLiveLeaseIntent
|
||||
) -> tuple[ObservatoryLiveLease, bool]:
|
||||
def request_live(self, intent: ObservatoryLiveLeaseIntent) -> tuple[ObservatoryLiveLease, bool]:
|
||||
"""Close recorded admission without allowing a monolith to delay live K1."""
|
||||
|
||||
created = False
|
||||
@@ -1357,9 +1321,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"scheduler did not release non-checkpointable replay resources"
|
||||
) from exc
|
||||
if (
|
||||
not isinstance(
|
||||
receipt, ObservatoryNonCheckpointableCancellationReceipt
|
||||
)
|
||||
not isinstance(receipt, ObservatoryNonCheckpointableCancellationReceipt)
|
||||
or receipt.request_sha256 != request.request_sha256
|
||||
):
|
||||
with self._transaction() as connection:
|
||||
@@ -1423,9 +1385,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"""Release live priority only on an exact explicit terminal trigger."""
|
||||
|
||||
_validate_pattern(lease_id, _LEASE_ID, "live lease id")
|
||||
_validate_pattern(
|
||||
terminal_trigger_id, _IDEMPOTENCY_KEY, "live terminal trigger id"
|
||||
)
|
||||
_validate_pattern(terminal_trigger_id, _IDEMPOTENCY_KEY, "live terminal trigger id")
|
||||
if outcome not in ("completed", "failed", "cancelled"):
|
||||
raise ValueError("live terminal outcome is invalid")
|
||||
terminal_request_sha256 = _sha256(
|
||||
@@ -1500,9 +1460,7 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._read_connection() as connection:
|
||||
row = self._open_live_lease_row(connection)
|
||||
if row is None:
|
||||
return ObservatoryRecordedAdmissionGate(
|
||||
blocked=False, lease_id=None, lease_state=None
|
||||
)
|
||||
return ObservatoryRecordedAdmissionGate(blocked=False, lease_id=None, lease_state=None)
|
||||
state = str(row["state"])
|
||||
if state == "pending":
|
||||
lease_state: Literal["pending", "active"] = "pending"
|
||||
@@ -1556,8 +1514,7 @@ class ObservatoryRecordedJobQueue:
|
||||
f"cannot terminate recorded job from {job.state}"
|
||||
)
|
||||
if state == "succeeded" and (
|
||||
job.preemption_requested
|
||||
or self._open_live_lease_row(connection) is not None
|
||||
job.preemption_requested or self._open_live_lease_row(connection) is not None
|
||||
):
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"cannot publish recorded success while live preemption is open"
|
||||
@@ -1599,14 +1556,10 @@ class ObservatoryRecordedJobQueue:
|
||||
job=self._get_job(connection, job_id),
|
||||
)
|
||||
|
||||
def _require_active_claim(
|
||||
self, job: ObservatoryRecordedJob, claim_token: str
|
||||
) -> None:
|
||||
def _require_active_claim(self, job: ObservatoryRecordedJob, claim_token: str) -> None:
|
||||
_validate_pattern(claim_token, _TOKEN, "claim token")
|
||||
if job.active_claim_token != claim_token:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim token is stale"
|
||||
)
|
||||
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim token is stale")
|
||||
|
||||
def _cancellation_request(
|
||||
self,
|
||||
@@ -1642,8 +1595,7 @@ class ObservatoryRecordedJobQueue:
|
||||
) -> None:
|
||||
with self._transaction() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_preemptions "
|
||||
"WHERE cancellation_request_id = ?",
|
||||
"SELECT * FROM observatory_recorded_preemptions WHERE cancellation_request_id = ?",
|
||||
(request.cancellation_request_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
@@ -1704,9 +1656,7 @@ class ObservatoryRecordedJobQueue:
|
||||
(receipt.receipt_sha256, now, request.cancellation_request_id),
|
||||
)
|
||||
|
||||
def _get_job(
|
||||
self, connection: sqlite3.Connection, job_id: str
|
||||
) -> ObservatoryRecordedJob:
|
||||
def _get_job(self, connection: sqlite3.Connection, job_id: str) -> ObservatoryRecordedJob:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs WHERE job_id = ?", (job_id,)
|
||||
).fetchone()
|
||||
@@ -1724,17 +1674,12 @@ class ObservatoryRecordedJobQueue:
|
||||
raise ObservatoryRecordedQueueNotFoundError(lease_id)
|
||||
return _live_lease_from_row(row)
|
||||
|
||||
def _open_live_lease_row(
|
||||
self, connection: sqlite3.Connection
|
||||
) -> sqlite3.Row | None:
|
||||
def _open_live_lease_row(self, connection: sqlite3.Connection) -> sqlite3.Row | None:
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observatory_live_leases "
|
||||
"WHERE state IN ('pending', 'active') LIMIT 2"
|
||||
"SELECT * FROM observatory_live_leases WHERE state IN ('pending', 'active') LIMIT 2"
|
||||
).fetchall()
|
||||
if len(rows) > 1:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"multiple live K1 leases are open"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("multiple live K1 leases are open")
|
||||
return None if not rows else rows[0]
|
||||
|
||||
def _require_capacity(
|
||||
@@ -1788,9 +1733,7 @@ class ObservatoryRecordedJobQueue:
|
||||
"SELECT name FROM pragma_table_info(?)", (table,)
|
||||
).fetchall()
|
||||
if len(columns) != column_count:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
f"{table} schema is incompatible"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError(f"{table} schema is incompatible")
|
||||
indexes = connection.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'index' "
|
||||
"AND name = 'observatory_one_open_live_lease'"
|
||||
@@ -1816,14 +1759,10 @@ class ObservatoryRecordedJobQueue:
|
||||
),
|
||||
):
|
||||
count = len(
|
||||
connection.execute(
|
||||
f"SELECT 1 FROM {table} LIMIT ?", (limit + 1,)
|
||||
).fetchall()
|
||||
connection.execute(f"SELECT 1 FROM {table} LIMIT ?", (limit + 1,)).fetchall()
|
||||
)
|
||||
if count > limit:
|
||||
raise ObservatoryRecordedQueueCapacityError(
|
||||
f"{label} quota is exceeded"
|
||||
)
|
||||
raise ObservatoryRecordedQueueCapacityError(f"{label} quota is exceeded")
|
||||
|
||||
def _validate_storage_paths(self, *, require_database: bool = False) -> None:
|
||||
paths = (
|
||||
@@ -1834,9 +1773,7 @@ class ObservatoryRecordedJobQueue:
|
||||
total_bytes = 0
|
||||
for path in paths:
|
||||
if path.is_symlink() or (path.exists() and not path.is_file()):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job queue path is unsafe"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job queue path is unsafe")
|
||||
if path.exists():
|
||||
total_bytes += path.stat().st_size
|
||||
if require_database and not self.database_path.is_file():
|
||||
@@ -1874,9 +1811,7 @@ class ObservatoryRecordedJobQueue:
|
||||
except ObservatoryRecordedQueueError:
|
||||
raise
|
||||
except (OSError, sqlite3.Error) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job queue read failed"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job queue read failed") from exc
|
||||
|
||||
@contextmanager
|
||||
def _connect(self) -> Iterator[sqlite3.Connection]:
|
||||
@@ -1889,9 +1824,7 @@ class ObservatoryRecordedJobQueue:
|
||||
try:
|
||||
connection.execute("PRAGMA foreign_keys = ON")
|
||||
connection.execute("PRAGMA synchronous = FULL")
|
||||
connection.execute(
|
||||
f"PRAGMA busy_timeout = {_SQLITE_BUSY_TIMEOUT_MILLISECONDS}"
|
||||
)
|
||||
connection.execute(f"PRAGMA busy_timeout = {_SQLITE_BUSY_TIMEOUT_MILLISECONDS}")
|
||||
connection.execute("PRAGMA journal_mode = WAL")
|
||||
yield connection
|
||||
finally:
|
||||
@@ -1944,9 +1877,7 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
source_session_id=row["source_session_id"],
|
||||
source_catalog_sha256=row["source_catalog_sha256"],
|
||||
source_bundle_sha256=row["source_bundle_sha256"],
|
||||
source_capability_manifest_sha256=row[
|
||||
"source_capability_manifest_sha256"
|
||||
],
|
||||
source_capability_manifest_sha256=row["source_capability_manifest_sha256"],
|
||||
setup_id=row["setup_id"],
|
||||
definition_id=row["definition_id"],
|
||||
definition_version=row["definition_version"],
|
||||
@@ -1982,9 +1913,7 @@ def _job_from_row(row: sqlite3.Row) -> ObservatoryRecordedJob:
|
||||
updated_at_utc=row["updated_at_utc"],
|
||||
)
|
||||
except (IndexError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored recorded job is invalid"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("stored recorded job is invalid") from exc
|
||||
|
||||
|
||||
def _live_lease_from_row(row: sqlite3.Row) -> ObservatoryLiveLease:
|
||||
@@ -2005,9 +1934,7 @@ def _live_lease_from_row(row: sqlite3.Row) -> ObservatoryLiveLease:
|
||||
terminated_at_utc=row["terminated_at_utc"],
|
||||
)
|
||||
except (IndexError, KeyError, TypeError, ValueError) as exc:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored live K1 lease is invalid"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueIntegrityError("stored live K1 lease is invalid") from exc
|
||||
|
||||
|
||||
def _job_identity_sha256(
|
||||
@@ -2019,9 +1946,7 @@ def _job_identity_sha256(
|
||||
"source_session_id": intent.source_session_id,
|
||||
"source_catalog_sha256": intent.source_catalog_sha256,
|
||||
"source_bundle_sha256": intent.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (
|
||||
intent.source_capability_manifest_sha256
|
||||
),
|
||||
"source_capability_manifest_sha256": (intent.source_capability_manifest_sha256),
|
||||
"setup_id": definition.setup_id,
|
||||
"definition_id": definition.definition_id,
|
||||
"definition_version": definition.definition_version,
|
||||
@@ -2100,11 +2025,7 @@ def _sha256(value: object) -> str:
|
||||
|
||||
|
||||
def _validate_quota(value: object, maximum: int, label: str) -> None:
|
||||
if (
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or not 1 <= value <= maximum
|
||||
):
|
||||
if not isinstance(value, int) or isinstance(value, bool) or not 1 <= value <= maximum:
|
||||
raise ValueError(f"{label} quota is invalid")
|
||||
|
||||
|
||||
@@ -2122,9 +2043,7 @@ def _validate_pattern(value: object, pattern: re.Pattern[str], label: str) -> No
|
||||
raise ValueError(f"{label} is invalid")
|
||||
|
||||
|
||||
def _validate_optional_pattern(
|
||||
value: object | None, pattern: re.Pattern[str], label: str
|
||||
) -> None:
|
||||
def _validate_optional_pattern(value: object | None, pattern: re.Pattern[str], label: str) -> None:
|
||||
if value is not None:
|
||||
_validate_pattern(value, pattern, label)
|
||||
|
||||
@@ -2146,11 +2065,7 @@ def _validate_timestamp(value: object, label: str) -> None:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{label} is invalid") from exc
|
||||
if (
|
||||
parsed.tzinfo is None
|
||||
or parsed.utcoffset() != timedelta(0)
|
||||
or not value.endswith("Z")
|
||||
):
|
||||
if parsed.tzinfo is None or parsed.utcoffset() != timedelta(0) or not value.endswith("Z"):
|
||||
raise ValueError(f"{label} must use UTC")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,978 @@
|
||||
"""Capability-based admission for portable recorded K1 laboratory runs.
|
||||
|
||||
The portable setup identity is deliberately independent from a session name.
|
||||
This module binds one concrete SessionStore snapshot to an allowlisted K1
|
||||
source profile and emits path-free, content-addressed documents for the Worker
|
||||
boundary. It does not enqueue work or expose host filesystem locations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from k1link.sessions.media import (
|
||||
CAMERA_ARCHIVE_SCHEMA,
|
||||
MAX_MEDIA_SUMMARY_BYTES,
|
||||
MAX_SAFE_INTEGER,
|
||||
RecordedMediaEpoch,
|
||||
RecordedMediaInspector,
|
||||
RecordedMediaManifest,
|
||||
)
|
||||
from k1link.sessions.models import (
|
||||
RecordedMediaArtifact,
|
||||
ReplayCommand,
|
||||
SessionArtifact,
|
||||
SessionDetail,
|
||||
SessionSource,
|
||||
)
|
||||
from k1link.sessions.store import SessionStore
|
||||
|
||||
PORTABLE_SOURCE_BUNDLE_SCHEMA: Final = "missioncore.portable-recorded-source-bundle/v1"
|
||||
PORTABLE_SOURCE_CAPABILITY_SCHEMA: Final = "missioncore.portable-recorded-source-capability/v1"
|
||||
PORTABLE_SOURCE_ADAPTER_SCHEMA: Final = "missioncore.portable-source-adapter/v1"
|
||||
PORTABLE_SOURCE_DOCUMENT_DIRECTORY: Final = "observatory-portable-source-contracts"
|
||||
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9.-]{2,127}$")
|
||||
_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_MEDIA_TYPE = re.compile(r'^[a-z0-9.+-]+/[a-z0-9.+-]+(?:; codecs="[A-Za-z0-9.]+")?$')
|
||||
_EPOCH = re.compile(r"^epoch-([1-9][0-9]*)$")
|
||||
_ALLOWED_MODALITIES: Final = frozenset({"point-cloud", "trajectory", "video"})
|
||||
_SEALED_ARTIFACT_STATES: Final = frozenset({"verified", "validated-structure"})
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class PortableSourceAdmissionError(RuntimeError):
|
||||
"""A recorded session does not satisfy the portable source contract."""
|
||||
|
||||
|
||||
class PortableSourceAdmissionIntegrityError(PortableSourceAdmissionError):
|
||||
"""A catalog, media, or persisted source identity changed."""
|
||||
|
||||
|
||||
class PortableSourceNotPreparedError(PortableSourceAdmissionError):
|
||||
"""A compatible source has no previously validated media sidecar."""
|
||||
|
||||
|
||||
class PortableSourceAdmissionStaleError(PortableSourceAdmissionIntegrityError):
|
||||
"""The catalog changed across a checked admission boundary."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RecordedK1SourceRequirements:
|
||||
"""Typed matcher owned by a portable RunDefinition.
|
||||
|
||||
``camera_init_sha256`` is the exact ISO-BMFF initialization segment for the
|
||||
admitted codec/resolution profile. Width and height are asserted by that
|
||||
immutable media-profile attestation; arbitrary init segments are rejected.
|
||||
Calibration remains an external, digest-bound rig profile until recordings
|
||||
carry their own calibration snapshot.
|
||||
"""
|
||||
|
||||
adapter_id: str
|
||||
adapter_version: int
|
||||
plugin_id: str
|
||||
archive_id: str
|
||||
required_modalities: tuple[str, ...]
|
||||
camera_source_id: str
|
||||
camera_semantic_channel_id: str
|
||||
camera_media_type: str
|
||||
camera_init_sha256: str
|
||||
expected_width: int
|
||||
expected_height: int
|
||||
calibration_slot: str
|
||||
calibration_sha256: str
|
||||
require_single_camera_epoch: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.adapter_id, _IDENTIFIER, "source adapter id")
|
||||
if (
|
||||
not isinstance(self.adapter_version, int)
|
||||
or isinstance(self.adapter_version, bool)
|
||||
or self.adapter_version < 1
|
||||
):
|
||||
raise ValueError("source adapter version is invalid")
|
||||
_pattern(self.plugin_id, _IDENTIFIER, "device plugin id")
|
||||
_pattern(self.archive_id, _IDENTIFIER, "archive id")
|
||||
if (
|
||||
not self.required_modalities
|
||||
or len(set(self.required_modalities)) != len(self.required_modalities)
|
||||
or not set(self.required_modalities).issubset(_ALLOWED_MODALITIES)
|
||||
or "video" not in self.required_modalities
|
||||
):
|
||||
raise ValueError("portable source modalities are invalid")
|
||||
_pattern(self.camera_source_id, _SOURCE_ID, "camera source id")
|
||||
_pattern(
|
||||
self.camera_semantic_channel_id,
|
||||
_IDENTIFIER,
|
||||
"camera semantic channel id",
|
||||
)
|
||||
_pattern(self.camera_media_type, _MEDIA_TYPE, "camera media type")
|
||||
_digest(self.camera_init_sha256, "camera init sha256")
|
||||
if (
|
||||
not isinstance(self.expected_width, int)
|
||||
or isinstance(self.expected_width, bool)
|
||||
or not isinstance(self.expected_height, int)
|
||||
or isinstance(self.expected_height, bool)
|
||||
or not 1 <= self.expected_width <= 16_384
|
||||
or not 1 <= self.expected_height <= 16_384
|
||||
):
|
||||
raise ValueError("camera dimensions are invalid")
|
||||
_pattern(self.calibration_slot, _SOURCE_ID, "calibration slot")
|
||||
_digest(self.calibration_sha256, "calibration sha256")
|
||||
if self.require_single_camera_epoch is not True:
|
||||
raise ValueError("portable source v1 requires one camera epoch")
|
||||
|
||||
def adapter_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_ADAPTER_SCHEMA,
|
||||
"adapter_id": self.adapter_id,
|
||||
"version": self.adapter_version,
|
||||
"source": {
|
||||
"plugin_id": self.plugin_id,
|
||||
"archive_id": self.archive_id,
|
||||
"required_modalities": list(self.required_modalities),
|
||||
"camera_source_id": self.camera_source_id,
|
||||
"camera_semantic_channel_id": self.camera_semantic_channel_id,
|
||||
},
|
||||
"camera_profile": {
|
||||
"media_type": self.camera_media_type,
|
||||
"init_sha256": self.camera_init_sha256,
|
||||
"width": self.expected_width,
|
||||
"height": self.expected_height,
|
||||
"attestation": "exact-isobmff-init-sha256",
|
||||
},
|
||||
"calibration": {
|
||||
"slot": self.calibration_slot,
|
||||
"sha256": self.calibration_sha256,
|
||||
"binding": "external-rig-profile",
|
||||
},
|
||||
"camera_epoch_policy": "exactly-one-complete-epoch",
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
@property
|
||||
def adapter_sha256(self) -> str:
|
||||
return _sha256(self.adapter_document())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedSourceCapability:
|
||||
"""Cheap catalog/summary attestation used by compatibility surfaces.
|
||||
|
||||
This is intentionally not a replay admission. In particular it carries
|
||||
no timeline or content-addressed source bundle because proving those facts
|
||||
requires the prepared replay sidecar and a full archive validation.
|
||||
"""
|
||||
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_adapter_sha256: str
|
||||
camera_segment_count: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.source_session_id, _SOURCE_ID, "source session id")
|
||||
_digest(self.source_catalog_sha256, "source catalog sha256")
|
||||
_digest(self.source_adapter_sha256, "source adapter sha256")
|
||||
if (
|
||||
not isinstance(self.camera_segment_count, int)
|
||||
or isinstance(self.camera_segment_count, bool)
|
||||
or not 1 <= self.camera_segment_count <= MAX_SAFE_INTEGER
|
||||
):
|
||||
raise ValueError("camera segment count is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.portable-recorded-source-probe/v1",
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"camera_segment_count": self.camera_segment_count,
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PortableRecordedSourceAdmission:
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_sha256: str
|
||||
frame_count: int
|
||||
timeline_start_seconds: float
|
||||
timeline_end_seconds: float
|
||||
camera_generation_sha256: str
|
||||
source_bundle: bytes
|
||||
capability_manifest: bytes
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.source_session_id, _SOURCE_ID, "source session id")
|
||||
for label, value in (
|
||||
("source catalog sha256", self.source_catalog_sha256),
|
||||
("source bundle sha256", self.source_bundle_sha256),
|
||||
(
|
||||
"source capability manifest sha256",
|
||||
self.source_capability_manifest_sha256,
|
||||
),
|
||||
("source adapter sha256", self.source_adapter_sha256),
|
||||
("camera generation sha256", self.camera_generation_sha256),
|
||||
):
|
||||
_digest(value, label)
|
||||
if self.frame_count < 1 or self.timeline_end_seconds <= self.timeline_start_seconds:
|
||||
raise ValueError("portable source timeline is invalid")
|
||||
if hashlib.sha256(self.source_bundle).hexdigest() != self.source_bundle_sha256:
|
||||
raise ValueError("source bundle bytes do not match their identity")
|
||||
if (
|
||||
hashlib.sha256(self.capability_manifest).hexdigest()
|
||||
!= self.source_capability_manifest_sha256
|
||||
):
|
||||
raise ValueError("capability bytes do not match their identity")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.portable-recorded-source-admission/v1",
|
||||
"source_session_id": self.source_session_id,
|
||||
"source_catalog_sha256": self.source_catalog_sha256,
|
||||
"source_bundle_sha256": self.source_bundle_sha256,
|
||||
"source_capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"source_adapter_sha256": self.source_adapter_sha256,
|
||||
"camera": {
|
||||
"generation_sha256": self.camera_generation_sha256,
|
||||
"frame_count": self.frame_count,
|
||||
"timeline_start_seconds": self.timeline_start_seconds,
|
||||
"timeline_end_seconds": self.timeline_end_seconds,
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
@property
|
||||
def identity_sha256(self) -> str:
|
||||
"""Content identity used to fence check-to-admit transitions."""
|
||||
|
||||
return _sha256(self.as_dict())
|
||||
|
||||
|
||||
class RecordedK1SourceAdmissionService:
|
||||
"""Bind compatible recorded K1 sessions without trusting their labels."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
data_dir: Path,
|
||||
session_store: SessionStore,
|
||||
media_inspector: RecordedMediaInspector,
|
||||
requirements: RecordedK1SourceRequirements,
|
||||
prepare_media: bool = True,
|
||||
) -> None:
|
||||
self.data_dir = data_dir.expanduser().resolve()
|
||||
if self.data_dir != session_store.data_dir:
|
||||
raise ValueError("portable admission and SessionStore roots disagree")
|
||||
self._session_store = session_store
|
||||
self._media_inspector = media_inspector
|
||||
self._prepare_media = prepare_media
|
||||
self.requirements = requirements
|
||||
self._document_root = self.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY
|
||||
|
||||
def probe(self, source_session_id: str) -> PortableRecordedSourceCapability:
|
||||
"""Prove cheap setup compatibility without replay/media preparation.
|
||||
|
||||
The probe reads one atomic catalog snapshot, the catalog's single
|
||||
recorded-video handle, and one bounded canonical camera summary. It
|
||||
deliberately never calls ``prepare_replay`` or ``RecordedMediaInspector``
|
||||
and never enters the archive's index or segment directory.
|
||||
"""
|
||||
|
||||
_pattern(source_session_id, _SOURCE_ID, "source session id")
|
||||
try:
|
||||
detail, catalog_sha256 = self._session_store.get_session_with_catalog_snapshot(
|
||||
source_session_id
|
||||
)
|
||||
recorded_media = self._session_store.list_recorded_media(source_session_id)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source capability could not be resolved"
|
||||
) from exc
|
||||
_digest(catalog_sha256, "source catalog sha256")
|
||||
selected_sources = self._verify_catalog(
|
||||
detail,
|
||||
expected_session_id=source_session_id,
|
||||
)
|
||||
media_artifact = self._verify_recorded_media_artifact(
|
||||
detail=detail,
|
||||
camera_source=selected_sources["video"],
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
segment_count = self._probe_camera_summary(media_artifact)
|
||||
return PortableRecordedSourceCapability(
|
||||
source_session_id=detail.summary.session_id,
|
||||
source_catalog_sha256=catalog_sha256,
|
||||
source_adapter_sha256=self.requirements.adapter_sha256,
|
||||
camera_segment_count=segment_count,
|
||||
)
|
||||
|
||||
def check(self, source_session_id: str) -> PortableRecordedSourceAdmission:
|
||||
return self._prepare(
|
||||
source_session_id,
|
||||
persist=False,
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
def admit(
|
||||
self,
|
||||
source_session_id: str,
|
||||
*,
|
||||
expected_admission_sha256: str | None = None,
|
||||
) -> PortableRecordedSourceAdmission:
|
||||
"""Persist only the source identity most recently admitted by a check.
|
||||
|
||||
``expected_admission_sha256`` is optional for existing internal callers.
|
||||
Portable queue binding supplies it so a changed SessionStore snapshot is
|
||||
rejected before either immutable source document is written.
|
||||
"""
|
||||
|
||||
if expected_admission_sha256 is not None:
|
||||
_digest(expected_admission_sha256, "expected source admission sha256")
|
||||
return self._prepare(
|
||||
source_session_id,
|
||||
persist=True,
|
||||
prepare_media=self._prepare_media,
|
||||
expected_admission_sha256=expected_admission_sha256,
|
||||
)
|
||||
|
||||
def _prepare(
|
||||
self,
|
||||
source_session_id: str,
|
||||
*,
|
||||
persist: bool,
|
||||
prepare_media: bool,
|
||||
expected_admission_sha256: str | None = None,
|
||||
) -> PortableRecordedSourceAdmission:
|
||||
_pattern(source_session_id, _SOURCE_ID, "source session id")
|
||||
try:
|
||||
detail, catalog_sha256 = self._session_store.get_session_with_catalog_snapshot(
|
||||
source_session_id
|
||||
)
|
||||
replay = self._session_store.prepare_replay(source_session_id)
|
||||
recorded_media = self._session_store.list_recorded_media(source_session_id)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source could not be resolved"
|
||||
) from exc
|
||||
_digest(catalog_sha256, "source catalog sha256")
|
||||
selected_sources = self._verify_catalog(
|
||||
detail,
|
||||
expected_session_id=source_session_id,
|
||||
)
|
||||
camera_source = selected_sources["video"]
|
||||
media_artifact = self._verify_recorded_media_artifact(
|
||||
detail=detail,
|
||||
camera_source=camera_source,
|
||||
recorded_media=recorded_media,
|
||||
)
|
||||
self._verify_replay(
|
||||
detail=detail,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
)
|
||||
try:
|
||||
media = (
|
||||
self._media_inspector.inspect(media_artifact, replay)
|
||||
if prepare_media
|
||||
else self._media_inspector.restore_prepared(media_artifact, replay)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera manifest failed validation"
|
||||
) from exc
|
||||
if media is None:
|
||||
raise PortableSourceNotPreparedError("recorded camera manifest has not been prepared")
|
||||
epoch = self._verify_media(
|
||||
media,
|
||||
expected_session_id=detail.summary.session_id,
|
||||
media_artifact=media_artifact,
|
||||
)
|
||||
source_bundle_document = self._source_bundle_document(
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha256,
|
||||
selected_sources=selected_sources,
|
||||
replay=replay,
|
||||
media=media,
|
||||
)
|
||||
source_bundle = _canonical_json(source_bundle_document)
|
||||
source_bundle_sha256 = hashlib.sha256(source_bundle).hexdigest()
|
||||
capability_document = self._capability_document(
|
||||
detail=detail,
|
||||
catalog_sha256=catalog_sha256,
|
||||
source_bundle_sha256=source_bundle_sha256,
|
||||
selected_sources=selected_sources,
|
||||
media=media,
|
||||
)
|
||||
capability_manifest = _canonical_json(capability_document)
|
||||
capability_sha256 = hashlib.sha256(capability_manifest).hexdigest()
|
||||
admission = PortableRecordedSourceAdmission(
|
||||
source_session_id=detail.summary.session_id,
|
||||
source_catalog_sha256=catalog_sha256,
|
||||
source_bundle_sha256=source_bundle_sha256,
|
||||
source_capability_manifest_sha256=capability_sha256,
|
||||
source_adapter_sha256=self.requirements.adapter_sha256,
|
||||
frame_count=len(epoch.segments),
|
||||
timeline_start_seconds=epoch.timeline_start_seconds,
|
||||
timeline_end_seconds=epoch.timeline_end_seconds,
|
||||
camera_generation_sha256=media.generation_sha256,
|
||||
source_bundle=source_bundle,
|
||||
capability_manifest=capability_manifest,
|
||||
)
|
||||
if (
|
||||
expected_admission_sha256 is not None
|
||||
and admission.identity_sha256 != expected_admission_sha256
|
||||
):
|
||||
raise PortableSourceAdmissionStaleError(
|
||||
"recorded source changed after its admission check"
|
||||
)
|
||||
if persist:
|
||||
try:
|
||||
final_detail, final_catalog_sha256 = (
|
||||
self._session_store.get_session_with_catalog_snapshot(source_session_id)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source catalog could not be rechecked"
|
||||
) from exc
|
||||
_digest(final_catalog_sha256, "source catalog sha256")
|
||||
if final_catalog_sha256 != catalog_sha256 or final_detail != detail:
|
||||
raise PortableSourceAdmissionStaleError(
|
||||
"recorded source catalog changed during admission"
|
||||
)
|
||||
_write_immutable_document(
|
||||
self._document_root,
|
||||
source_bundle_sha256,
|
||||
source_bundle,
|
||||
)
|
||||
_write_immutable_document(
|
||||
self._document_root,
|
||||
capability_sha256,
|
||||
capability_manifest,
|
||||
)
|
||||
return admission
|
||||
|
||||
def _verify_catalog(
|
||||
self,
|
||||
detail: SessionDetail,
|
||||
*,
|
||||
expected_session_id: str,
|
||||
) -> dict[str, SessionSource]:
|
||||
expected = self.requirements
|
||||
summary = detail.summary
|
||||
if (
|
||||
summary.session_id != expected_session_id
|
||||
or summary.lab is not None
|
||||
or summary.status != "ready"
|
||||
or not summary.replayable
|
||||
or detail.plugin_id != expected.plugin_id
|
||||
or detail.archive_id != expected.archive_id
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"session is not an admitted recorded K1 source"
|
||||
)
|
||||
if not set(expected.required_modalities).issubset(
|
||||
summary.modalities
|
||||
) or summary.source_count != len(detail.sources):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"required recorded source modalities are unavailable"
|
||||
)
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
selected: dict[str, SessionSource] = {}
|
||||
for modality in expected.required_modalities:
|
||||
matches = tuple(source for source in detail.sources if source.modality == modality)
|
||||
if len(matches) != 1:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded modality does not have one canonical source"
|
||||
)
|
||||
source = matches[0]
|
||||
if source.status != "recorded" or not source.seekable:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source is not sealed and seekable"
|
||||
)
|
||||
artifact = artifacts.get(source.artifact_id)
|
||||
if artifact is None or artifact.integrity_status not in _SEALED_ARTIFACT_STATES:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded source artifact is not sealed"
|
||||
)
|
||||
if modality != "video" and artifact.sha256 is None:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial source artifact has no content identity"
|
||||
)
|
||||
selected[modality] = source
|
||||
camera = selected["video"]
|
||||
recorded_video_artifacts = tuple(
|
||||
artifact for artifact in detail.artifacts if artifact.kind == "recorded-video"
|
||||
)
|
||||
if (
|
||||
camera.source_id != expected.camera_source_id
|
||||
or camera.semantic_channel_id != expected.camera_semantic_channel_id
|
||||
or len(recorded_video_artifacts) != 1
|
||||
or recorded_video_artifacts[0].artifact_id != camera.artifact_id
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera capability does not match the setup"
|
||||
)
|
||||
return selected
|
||||
|
||||
def _verify_recorded_media_artifact(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
camera_source: SessionSource,
|
||||
recorded_media: tuple[RecordedMediaArtifact, ...],
|
||||
) -> RecordedMediaArtifact:
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
catalog_artifact = artifacts.get(camera_source.artifact_id)
|
||||
matching_media = tuple(
|
||||
item for item in recorded_media if item.artifact_id == camera_source.artifact_id
|
||||
)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or catalog_artifact.kind != "recorded-video"
|
||||
or len(recorded_media) != 1
|
||||
or len(matching_media) != 1
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera artifact is unavailable")
|
||||
media_artifact = matching_media[0]
|
||||
if (
|
||||
media_artifact.session_id != detail.summary.session_id
|
||||
or media_artifact.artifact_id != catalog_artifact.artifact_id
|
||||
or media_artifact.byte_length != catalog_artifact.byte_length
|
||||
or media_artifact.byte_length < 1
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera artifact identity disagrees with the catalog"
|
||||
)
|
||||
_pattern(
|
||||
media_artifact.public_source_id,
|
||||
_SOURCE_ID,
|
||||
"recorded camera public source id",
|
||||
)
|
||||
return media_artifact
|
||||
|
||||
def _verify_replay(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
replay: ReplayCommand,
|
||||
) -> None:
|
||||
if (
|
||||
replay.session_id != detail.summary.session_id
|
||||
or replay.plugin_id != detail.plugin_id
|
||||
or replay.plugin_id != self.requirements.plugin_id
|
||||
or replay.speed != 1.0
|
||||
or replay.loop
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay is bound to another source contract"
|
||||
)
|
||||
catalog_artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
replay_ids = tuple(artifact.artifact_id for artifact in replay.artifacts)
|
||||
if not replay_ids or len(set(replay_ids)) != len(replay_ids):
|
||||
raise PortableSourceAdmissionIntegrityError("spatial replay members are not unique")
|
||||
for replay_artifact in replay.artifacts:
|
||||
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||
or replay_artifact.media_type != catalog_artifact.media_type
|
||||
or replay_artifact.file_byte_length != catalog_artifact.byte_length
|
||||
or not 1 <= replay_artifact.replay_byte_length <= replay_artifact.file_byte_length
|
||||
or replay_artifact.expected_sha256 != catalog_artifact.sha256
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"spatial replay member disagrees with the catalog"
|
||||
)
|
||||
required_spatial_artifact_ids = {
|
||||
selected_sources[modality].artifact_id
|
||||
for modality in self.requirements.required_modalities
|
||||
if modality != "video"
|
||||
}
|
||||
if (
|
||||
replay.primary_artifact_id not in replay_ids
|
||||
or not required_spatial_artifact_ids.issubset(replay_ids)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"required spatial artifacts are absent from replay"
|
||||
)
|
||||
|
||||
def _probe_camera_summary(
|
||||
self,
|
||||
media_artifact: RecordedMediaArtifact,
|
||||
) -> int:
|
||||
summary, epoch_ordinal = _read_canonical_camera_summary(media_artifact.source_path)
|
||||
segment_count = summary.get("segment_count")
|
||||
if (
|
||||
summary.get("schema_version") != CAMERA_ARCHIVE_SCHEMA
|
||||
or summary.get("source_id") != self.requirements.camera_source_id
|
||||
or summary.get("codec_epoch") != epoch_ordinal
|
||||
or summary.get("status") != "complete"
|
||||
or summary.get("synchronization") != "host-arrival-best-effort"
|
||||
or summary.get("failure_code") is not None
|
||||
or summary.get("init_sha256") != self.requirements.camera_init_sha256
|
||||
or not isinstance(segment_count, int)
|
||||
or isinstance(segment_count, bool)
|
||||
or not 1 <= segment_count <= MAX_SAFE_INTEGER
|
||||
or summary.get("entry_count") != segment_count
|
||||
or summary.get("media_segment_count") != segment_count
|
||||
or summary.get("commit_policy") != "per-segment-fsync"
|
||||
or summary.get("artifacts")
|
||||
!= {
|
||||
"init": "init.mp4",
|
||||
"segments": "segments",
|
||||
"index": "index.jsonl",
|
||||
}
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is incompatible")
|
||||
return segment_count
|
||||
|
||||
def _verify_media(
|
||||
self,
|
||||
manifest: RecordedMediaManifest,
|
||||
*,
|
||||
expected_session_id: str,
|
||||
media_artifact: RecordedMediaArtifact,
|
||||
) -> RecordedMediaEpoch:
|
||||
expected = self.requirements
|
||||
if (
|
||||
manifest.session_id != expected_session_id
|
||||
or manifest.session_id != media_artifact.session_id
|
||||
or manifest.artifact_id != media_artifact.artifact_id
|
||||
or manifest.public_source_id != media_artifact.public_source_id
|
||||
or manifest.byte_length != media_artifact.byte_length
|
||||
or len(manifest.epochs) != 1
|
||||
or manifest.synchronization != "host-arrival-best-effort"
|
||||
or not _SHA256.fullmatch(manifest.generation_sha256)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera epoch topology is unsupported"
|
||||
)
|
||||
epoch = manifest.epochs[0]
|
||||
if (
|
||||
epoch.media_type != expected.camera_media_type
|
||||
or epoch.init_sha256 != expected.camera_init_sha256
|
||||
or not epoch.segments
|
||||
or epoch.timeline_end_seconds <= epoch.timeline_start_seconds
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera media profile is incompatible"
|
||||
)
|
||||
return epoch
|
||||
|
||||
def _source_bundle_document(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
catalog_sha256: str,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
replay: ReplayCommand,
|
||||
media: RecordedMediaManifest,
|
||||
) -> dict[str, object]:
|
||||
epoch = media.epochs[0]
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_BUNDLE_SCHEMA,
|
||||
"source_session_id": detail.summary.session_id,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"plugin_id": detail.plugin_id,
|
||||
"archive_id": detail.archive_id,
|
||||
"source_adapter": {
|
||||
"id": self.requirements.adapter_id,
|
||||
"version": self.requirements.adapter_version,
|
||||
"sha256": self.requirements.adapter_sha256,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
**selected_sources[modality].as_dict(),
|
||||
"artifact": _artifact_document(
|
||||
artifacts[selected_sources[modality].artifact_id]
|
||||
),
|
||||
}
|
||||
for modality in self.requirements.required_modalities
|
||||
],
|
||||
"spatial_replay": {
|
||||
"primary_artifact_id": replay.primary_artifact_id,
|
||||
"members": [
|
||||
{
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"media_type": artifact.media_type,
|
||||
"byte_length": artifact.file_byte_length,
|
||||
"replay_byte_length": artifact.replay_byte_length,
|
||||
"sha256": artifact.expected_sha256,
|
||||
}
|
||||
for artifact in replay.artifacts
|
||||
],
|
||||
"timeline_origin_epoch_ns": replay.timeline_origin_epoch_ns,
|
||||
"timeline_origin_monotonic_ns": replay.timeline_origin_monotonic_ns,
|
||||
},
|
||||
"camera": {
|
||||
"artifact_id": media.artifact_id,
|
||||
"public_source_id": media.public_source_id,
|
||||
"generation_sha256": media.generation_sha256,
|
||||
"synchronization": media.synchronization,
|
||||
"epoch": {
|
||||
"ordinal": epoch.ordinal,
|
||||
"media_type": epoch.media_type,
|
||||
"init": {
|
||||
"byte_length": epoch.init_byte_length,
|
||||
"sha256": epoch.init_sha256,
|
||||
},
|
||||
"timeline_start_seconds": epoch.timeline_start_seconds,
|
||||
"timeline_end_seconds": epoch.timeline_end_seconds,
|
||||
"segments": [
|
||||
{
|
||||
"sequence": segment.sequence,
|
||||
"byte_length": segment.byte_length,
|
||||
"sha256": segment.sha256,
|
||||
"random_access": segment.random_access,
|
||||
"end_time_seconds": segment.end_time_seconds,
|
||||
}
|
||||
for segment in epoch.segments
|
||||
],
|
||||
},
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
def _capability_document(
|
||||
self,
|
||||
*,
|
||||
detail: SessionDetail,
|
||||
catalog_sha256: str,
|
||||
source_bundle_sha256: str,
|
||||
selected_sources: dict[str, SessionSource],
|
||||
media: RecordedMediaManifest,
|
||||
) -> dict[str, object]:
|
||||
epoch = media.epochs[0]
|
||||
return {
|
||||
"schema_version": PORTABLE_SOURCE_CAPABILITY_SCHEMA,
|
||||
"source_session_id": detail.summary.session_id,
|
||||
"source_catalog_sha256": catalog_sha256,
|
||||
"source_bundle_sha256": source_bundle_sha256,
|
||||
"source_adapter_sha256": self.requirements.adapter_sha256,
|
||||
"modalities": [
|
||||
{
|
||||
"modality": modality,
|
||||
"source_id": selected_sources[modality].source_id,
|
||||
"semantic_channel_id": (selected_sources[modality].semantic_channel_id),
|
||||
"seekable": True,
|
||||
}
|
||||
for modality in self.requirements.required_modalities
|
||||
],
|
||||
"camera_profile": {
|
||||
"media_type": epoch.media_type,
|
||||
"init_sha256": epoch.init_sha256,
|
||||
"width": self.requirements.expected_width,
|
||||
"height": self.requirements.expected_height,
|
||||
"profile_attestation": "exact-isobmff-init-sha256",
|
||||
"generation_sha256": media.generation_sha256,
|
||||
"frame_count": len(epoch.segments),
|
||||
"timeline_start_seconds": epoch.timeline_start_seconds,
|
||||
"timeline_end_seconds": epoch.timeline_end_seconds,
|
||||
},
|
||||
"calibration": {
|
||||
"slot": self.requirements.calibration_slot,
|
||||
"sha256": self.requirements.calibration_sha256,
|
||||
"binding": "external-rig-profile",
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
def _read_canonical_camera_summary(
|
||||
source_path: Path,
|
||||
) -> tuple[dict[str, object], int]:
|
||||
"""Read one direct epoch summary through bounded no-follow descriptors."""
|
||||
|
||||
source_descriptor = -1
|
||||
epoch_descriptor = -1
|
||||
summary_descriptor = -1
|
||||
try:
|
||||
directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
|
||||
source_descriptor = os.open(source_path, directory_flags)
|
||||
if not stat.S_ISDIR(os.fstat(source_descriptor).st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera source is not a directory")
|
||||
epoch_name: str | None = None
|
||||
epoch_ordinal: int | None = None
|
||||
with os.scandir(source_descriptor) as entries:
|
||||
for entry in entries:
|
||||
match = _EPOCH.fullmatch(entry.name)
|
||||
if (
|
||||
epoch_name is not None
|
||||
or match is None
|
||||
or not entry.is_dir(follow_symlinks=False)
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera source does not have one canonical epoch"
|
||||
)
|
||||
epoch_name = entry.name
|
||||
epoch_ordinal = int(match.group(1))
|
||||
if epoch_name is None or epoch_ordinal is None:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera source does not have one canonical epoch"
|
||||
)
|
||||
epoch_descriptor = os.open(
|
||||
epoch_name,
|
||||
directory_flags,
|
||||
dir_fd=source_descriptor,
|
||||
)
|
||||
if not stat.S_ISDIR(os.fstat(epoch_descriptor).st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera epoch is not a directory")
|
||||
summary_descriptor = os.open(
|
||||
"summary.json",
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
||||
dir_fd=epoch_descriptor,
|
||||
)
|
||||
before = os.fstat(summary_descriptor)
|
||||
if not stat.S_ISREG(before.st_mode) or not 1 <= before.st_size <= MAX_MEDIA_SUMMARY_BYTES:
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is outside bounds")
|
||||
chunks: list[bytes] = []
|
||||
remaining = before.st_size
|
||||
while remaining:
|
||||
chunk = os.read(summary_descriptor, remaining)
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
payload = b"".join(chunks)
|
||||
after = os.fstat(summary_descriptor)
|
||||
if len(payload) != before.st_size or (
|
||||
after.st_dev,
|
||||
after.st_ino,
|
||||
after.st_size,
|
||||
after.st_mtime_ns,
|
||||
) != (
|
||||
before.st_dev,
|
||||
before.st_ino,
|
||||
before.st_size,
|
||||
before.st_mtime_ns,
|
||||
):
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary changed while it was read"
|
||||
)
|
||||
try:
|
||||
value = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary is invalid"
|
||||
) from exc
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise PortableSourceAdmissionIntegrityError("recorded camera summary is not an object")
|
||||
return value, epoch_ordinal
|
||||
except PortableSourceAdmissionIntegrityError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"recorded camera summary is unavailable"
|
||||
) from exc
|
||||
finally:
|
||||
for descriptor in (
|
||||
summary_descriptor,
|
||||
epoch_descriptor,
|
||||
source_descriptor,
|
||||
):
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _artifact_document(artifact: SessionArtifact) -> dict[str, object]:
|
||||
return {
|
||||
"artifact_id": artifact.artifact_id,
|
||||
"kind": artifact.kind,
|
||||
"media_type": artifact.media_type,
|
||||
"byte_length": artifact.byte_length,
|
||||
"sha256": artifact.sha256,
|
||||
"integrity_status": artifact.integrity_status,
|
||||
}
|
||||
|
||||
|
||||
def _write_immutable_document(root: Path, digest: str, payload: bytes) -> None:
|
||||
_digest(digest, "document sha256")
|
||||
if hashlib.sha256(payload).hexdigest() != digest:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"immutable document payload has another identity"
|
||||
)
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
metadata = root.lstat()
|
||||
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||
raise PortableSourceAdmissionIntegrityError("portable source document root is unsafe")
|
||||
destination = root / f"{digest}.json"
|
||||
if destination.exists():
|
||||
existing = destination.read_bytes()
|
||||
if existing != payload:
|
||||
raise PortableSourceAdmissionIntegrityError(
|
||||
"immutable source document identity collided"
|
||||
)
|
||||
return
|
||||
temporary = root / f".tmp-{secrets.token_hex(16)}"
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
published = False
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, destination)
|
||||
_fsync_directory(root)
|
||||
published = True
|
||||
finally:
|
||||
if not published:
|
||||
with suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: object) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _pattern(value: object, pattern: re.Pattern[str], label: str) -> str:
|
||||
if not isinstance(value, str) or pattern.fullmatch(value) is None:
|
||||
raise ValueError(f"{label} is invalid")
|
||||
return value
|
||||
@@ -0,0 +1,695 @@
|
||||
"""Transport-agnostic Worker 006 core for sealed Observatory jobs.
|
||||
|
||||
The agent accepts only the durable, path-free recorded-job projection. It
|
||||
validates every identity sealed by Mission Core, resolves an executor from a
|
||||
local four-digest allowlist, and passes a typed job to that executor. Server
|
||||
payloads can never provide commands, paths, environment variables, images, or
|
||||
other executable instructions.
|
||||
|
||||
Network authentication, polling cadence, local CAS resolution, ML runtimes,
|
||||
and deployment are deliberately outside this module. They are supplied by a
|
||||
transport and an executor adapter at the Worker boundary.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Final, Literal, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA,
|
||||
OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
)
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||
|
||||
_JOB_ID_PATTERN: Final = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN: Final = r"^[a-f0-9]{64}$"
|
||||
_CLAIM_REQUEST_ID_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
|
||||
_SESSION_ID_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
_IDENTIFIER_PATTERN: Final = r"^[a-z][a-z0-9-]{2,95}$"
|
||||
_SHA256_PATTERN: Final = r"^[a-f0-9]{64}$"
|
||||
_TIMESTAMP_PATTERN: Final = (
|
||||
r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}"
|
||||
r"(?:\.[0-9]{1,9})?(?:Z|[+-][0-9]{2}:[0-9]{2})$"
|
||||
)
|
||||
_CLAIM_REQUEST_ID = re.compile(_CLAIM_REQUEST_ID_PATTERN)
|
||||
|
||||
type WorkerCycleState = Literal[
|
||||
"empty",
|
||||
"deferred",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"rejected",
|
||||
]
|
||||
type RecordedJobWireState = Literal[
|
||||
"accepted",
|
||||
"queued",
|
||||
"claimed",
|
||||
"running",
|
||||
"paused",
|
||||
"preemption-pending",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"reconciliation-required",
|
||||
]
|
||||
|
||||
Sha256 = Annotated[str, Field(pattern=_SHA256_PATTERN)]
|
||||
Identifier = Annotated[
|
||||
str,
|
||||
Field(min_length=3, max_length=96, pattern=_IDENTIFIER_PATTERN),
|
||||
]
|
||||
SessionId = Annotated[
|
||||
str,
|
||||
Field(min_length=1, max_length=128, pattern=_SESSION_ID_PATTERN),
|
||||
]
|
||||
Timestamp = Annotated[
|
||||
str,
|
||||
Field(min_length=20, max_length=64, pattern=_TIMESTAMP_PATTERN),
|
||||
]
|
||||
|
||||
|
||||
class ObservatoryWorkerAgentError(RuntimeError):
|
||||
"""Base error for the local Worker 006 agent core."""
|
||||
|
||||
|
||||
class ObservatoryWorkerAgentBusyError(ObservatoryWorkerAgentError):
|
||||
"""A second poll cycle attempted to overlap the active claim."""
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRejectedError(ObservatoryWorkerAgentError):
|
||||
"""A transport supplied an unknown, spoofed, or corrupted claim."""
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError):
|
||||
"""No local adapter matches the exact sealed executor identity."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorIdentity:
|
||||
"""The only identity that may select executable Worker code."""
|
||||
|
||||
release_sha256: str
|
||||
image_sha256: str
|
||||
model_manifest_sha256: str
|
||||
resource_profile_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for label, value in (
|
||||
("executor release", self.release_sha256),
|
||||
("executor image", self.image_sha256),
|
||||
("model manifest", self.model_manifest_sha256),
|
||||
("resource profile", self.resource_profile_sha256),
|
||||
):
|
||||
if re.fullmatch(_SHA256_PATTERN, value) is None:
|
||||
raise ValueError(f"{label} SHA-256 is invalid")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SealedObservatoryRecordedJob:
|
||||
"""Validated path-free job passed to one local executor adapter."""
|
||||
|
||||
job_id: str
|
||||
request_sha256: str
|
||||
identity_sha256: str
|
||||
source_session_id: str
|
||||
source_catalog_sha256: str
|
||||
source_bundle_sha256: str
|
||||
source_capability_manifest_sha256: str
|
||||
source_adapter_id: str
|
||||
source_adapter_version: int
|
||||
source_adapter_sha256: str
|
||||
setup_id: str
|
||||
definition_id: str
|
||||
definition_version: int
|
||||
definition_sha256: str
|
||||
executor_release_id: str
|
||||
executor_identity: ObservatoryWorkerExecutorIdentity
|
||||
model_release_ids: tuple[str, ...]
|
||||
resource_profile_id: str
|
||||
checkpoint_policy: Literal["cooperative", "non-checkpointable"]
|
||||
allowed_checkpoints: tuple[str, ...]
|
||||
claim_generation: int
|
||||
restart_from_zero: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutionResult:
|
||||
"""Content-addressed result identity returned by a local executor."""
|
||||
|
||||
result_id: str
|
||||
result_sha256: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if re.fullmatch(_SESSION_ID_PATTERN, self.result_id) is None:
|
||||
raise ValueError("Worker result id is invalid")
|
||||
if re.fullmatch(_SHA256_PATTERN, self.result_sha256) is None:
|
||||
raise ValueError("Worker result SHA-256 is invalid")
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutor(Protocol):
|
||||
"""A local implementation selected only by its sealed digest tuple."""
|
||||
|
||||
def execute(
|
||||
self,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> ObservatoryWorkerExecutionResult: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorRegistration:
|
||||
identity: ObservatoryWorkerExecutorIdentity
|
||||
adapter: ObservatoryWorkerExecutor
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerExecutorRegistry:
|
||||
"""In-memory local executor allowlist; it never accepts server code."""
|
||||
|
||||
registrations: tuple[ObservatoryWorkerExecutorRegistration, ...]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
identities = [registration.identity for registration in self.registrations]
|
||||
if len(identities) != len(set(identities)):
|
||||
raise ValueError("Worker executor identities must be unique")
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
identity: ObservatoryWorkerExecutorIdentity,
|
||||
) -> ObservatoryWorkerExecutor:
|
||||
for registration in self.registrations:
|
||||
if registration.identity == identity:
|
||||
return registration.adapter
|
||||
raise ObservatoryWorkerExecutorUnavailableError(
|
||||
"exact executor identity is not locally allowlisted"
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerTransport(Protocol):
|
||||
"""State-transition port implemented by HTTP, IPC, or a test transport."""
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
claim_request_id: str,
|
||||
) -> Mapping[str, object] | None: ...
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def succeed(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
result_id: str,
|
||||
result_sha256: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
def fail(
|
||||
self,
|
||||
*,
|
||||
claimant_id: str,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
error_code: str,
|
||||
message: str,
|
||||
) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerCycleReport:
|
||||
state: WorkerCycleState
|
||||
claim_request_id: str
|
||||
job_id: str | None = None
|
||||
result_id: str | None = None
|
||||
failure_code: str | None = None
|
||||
|
||||
|
||||
class _StrictPayload(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
|
||||
class _AuthorityPayload(_StrictPayload):
|
||||
commands_enabled: Literal[False]
|
||||
actuation_allowed: Literal[False]
|
||||
navigation_or_safety_accepted: Literal[False]
|
||||
production_accepted: Literal[False]
|
||||
|
||||
|
||||
class _SourceAdapterPayload(_StrictPayload):
|
||||
adapter_id: Identifier
|
||||
version: int = Field(ge=1)
|
||||
adapter_sha256: Sha256
|
||||
|
||||
|
||||
class _SourcePayload(_StrictPayload):
|
||||
session_id: SessionId
|
||||
catalog_sha256: Sha256
|
||||
bundle_sha256: Sha256
|
||||
capability_manifest_sha256: Sha256
|
||||
adapter: _SourceAdapterPayload
|
||||
|
||||
|
||||
class _SetupPayload(_StrictPayload):
|
||||
setup_id: Identifier
|
||||
definition_id: Identifier
|
||||
definition_version: int = Field(ge=1)
|
||||
definition_sha256: Sha256
|
||||
|
||||
|
||||
class _ExecutorPayload(_StrictPayload):
|
||||
release_id: Identifier
|
||||
release_sha256: Sha256
|
||||
image_sha256: Sha256
|
||||
model_release_ids: list[Identifier] = Field(max_length=32)
|
||||
learned_models: list[Identifier] = Field(max_length=32)
|
||||
model_manifest_sha256: Sha256
|
||||
resource_profile_id: Identifier
|
||||
resource_profile_sha256: Sha256
|
||||
|
||||
|
||||
class _CheckpointPolicyPayload(_StrictPayload):
|
||||
mode: Literal["cooperative", "non-checkpointable"]
|
||||
allowed_checkpoints: list[Identifier] = Field(max_length=64)
|
||||
last_checkpoint_id: Identifier | None
|
||||
|
||||
|
||||
class _PriorityPayload(_StrictPayload):
|
||||
class_: Literal["recorded"] = Field(alias="class")
|
||||
rank: Literal[100]
|
||||
server_owned: Literal[True]
|
||||
|
||||
|
||||
class _ResultPayload(_StrictPayload):
|
||||
result_id: SessionId
|
||||
sha256: Sha256
|
||||
|
||||
|
||||
class _TerminalPayload(_StrictPayload):
|
||||
code: Identifier
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
class _RecordedJobPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job/v1"]
|
||||
job_id: str = Field(pattern=_JOB_ID_PATTERN)
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
request_sha256: Sha256
|
||||
identity_sha256: Sha256
|
||||
submission_receipt_sha256: Sha256
|
||||
source: _SourcePayload
|
||||
setup: _SetupPayload
|
||||
executor: _ExecutorPayload
|
||||
checkpoint_policy: _CheckpointPolicyPayload
|
||||
priority: _PriorityPayload
|
||||
state: RecordedJobWireState
|
||||
preemption_requested: bool
|
||||
restart_from_zero: bool
|
||||
preemption_receipt_sha256: Sha256 | None
|
||||
claim_generation: int = Field(ge=0)
|
||||
result: _ResultPayload | None
|
||||
terminal: _TerminalPayload | None
|
||||
created_at_utc: Timestamp
|
||||
updated_at_utc: Timestamp
|
||||
authority: _AuthorityPayload
|
||||
|
||||
|
||||
class _RecordedClaimPayload(_StrictPayload):
|
||||
schema_version: Literal["missioncore.observatory-recorded-job-claim/v1"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
request_sha256: Sha256
|
||||
claimant_id: Identifier
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
job: _RecordedJobPayload
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ValidatedClaim:
|
||||
claim_request_id: str
|
||||
claim_token: str
|
||||
job: SealedObservatoryRecordedJob
|
||||
|
||||
|
||||
class ObservatoryWorkerAgent:
|
||||
"""Runs at most one sealed recorded-job claim at a time on Worker 006."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
transport: ObservatoryWorkerTransport,
|
||||
executors: ObservatoryWorkerExecutorRegistry,
|
||||
claim_request_id_factory: Callable[[], str] | None = None,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._executors = executors
|
||||
self._claim_request_id_factory = claim_request_id_factory or _default_claim_request_id
|
||||
self._cycle_lock = threading.Lock()
|
||||
|
||||
def run_once(self) -> ObservatoryWorkerCycleReport:
|
||||
"""Claim, validate, execute, and seal one durable job if available."""
|
||||
|
||||
if not self._cycle_lock.acquire(blocking=False):
|
||||
raise ObservatoryWorkerAgentBusyError("Worker 006 already owns an active claim cycle")
|
||||
try:
|
||||
return self._run_once_locked()
|
||||
finally:
|
||||
self._cycle_lock.release()
|
||||
|
||||
def _run_once_locked(self) -> ObservatoryWorkerCycleReport:
|
||||
claim_request_id = self._claim_request_id_factory()
|
||||
if _CLAIM_REQUEST_ID.fullmatch(claim_request_id) is None:
|
||||
raise ValueError("Worker claim request id is invalid")
|
||||
payload = self._transport.claim_next(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
if payload is None:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="empty",
|
||||
claim_request_id=claim_request_id,
|
||||
)
|
||||
try:
|
||||
claim = _validate_claim(payload, claim_request_id=claim_request_id)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="rejected",
|
||||
claim_request_id=claim_request_id,
|
||||
failure_code="claim-rejected",
|
||||
)
|
||||
|
||||
try:
|
||||
adapter = self._executors.resolve(claim.job.executor_identity)
|
||||
except ObservatoryWorkerExecutorUnavailableError:
|
||||
failure_code = "executor-not-allowlisted"
|
||||
acknowledgement = self._transport.fail(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
error_code=failure_code,
|
||||
message="Exact executor identity is not installed on Worker 006.",
|
||||
)
|
||||
_validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="failed",
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="failed",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code=failure_code,
|
||||
)
|
||||
|
||||
started_payload = self._transport.start(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
)
|
||||
started = _validate_transition_acknowledgement(
|
||||
started_payload,
|
||||
expected_job=claim.job,
|
||||
expected_state=("running", "paused"),
|
||||
)
|
||||
if started.state == "paused":
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="deferred",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
)
|
||||
|
||||
try:
|
||||
result = adapter.execute(claim.job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
failure_code = "executor-error"
|
||||
acknowledgement = self._transport.fail(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
error_code=failure_code,
|
||||
message=_bounded_executor_failure(exc),
|
||||
)
|
||||
_validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="failed",
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="failed",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
failure_code=failure_code,
|
||||
)
|
||||
|
||||
acknowledgement = self._transport.succeed(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=claim.job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
result_id=result.result_id,
|
||||
result_sha256=result.result_sha256,
|
||||
)
|
||||
succeeded = _validate_transition_acknowledgement(
|
||||
acknowledgement,
|
||||
expected_job=claim.job,
|
||||
expected_state="succeeded",
|
||||
)
|
||||
if succeeded.result is None or (
|
||||
succeeded.result.result_id != result.result_id
|
||||
or succeeded.result.sha256 != result.result_sha256
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker success acknowledgement changed result identity"
|
||||
)
|
||||
return ObservatoryWorkerCycleReport(
|
||||
state="succeeded",
|
||||
claim_request_id=claim_request_id,
|
||||
job_id=claim.job.job_id,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _validate_claim(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
claim_request_id: str,
|
||||
) -> _ValidatedClaim:
|
||||
try:
|
||||
claim = _RecordedClaimPayload.model_validate(dict(payload))
|
||||
if claim.claim_request_id != claim_request_id:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim request identity changed")
|
||||
if claim.claimant_id != WORKER_006_CONTOUR_ID:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim belongs to another claimant")
|
||||
expected_claim_request_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_CLAIM_SCHEMA,
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": WORKER_006_CONTOUR_ID,
|
||||
}
|
||||
)
|
||||
if claim.request_sha256 != expected_claim_request_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim request digest changed")
|
||||
if claim.job.state != "claimed" or claim.job.claim_generation < 1:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim job is not in a claimed generation"
|
||||
)
|
||||
if claim.job.result is not None or claim.job.terminal is not None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim already carries a terminal outcome"
|
||||
)
|
||||
job = _seal_job(claim.job)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
raise
|
||||
except (TypeError, ValueError, ValidationError) as exc:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim payload violates the sealed protocol"
|
||||
) from exc
|
||||
return _ValidatedClaim(
|
||||
claim_request_id=claim.claim_request_id,
|
||||
claim_token=claim.claim_token,
|
||||
job=job,
|
||||
)
|
||||
|
||||
|
||||
def _seal_job(payload: _RecordedJobPayload) -> SealedObservatoryRecordedJob:
|
||||
if payload.executor.learned_models != payload.executor.model_release_ids:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim learned-model identities changed")
|
||||
if len(set(payload.executor.model_release_ids)) != len(payload.executor.model_release_ids):
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim model identities are not unique")
|
||||
if len(set(payload.checkpoint_policy.allowed_checkpoints)) != len(
|
||||
payload.checkpoint_policy.allowed_checkpoints
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim checkpoint identities are not unique"
|
||||
)
|
||||
if (
|
||||
payload.checkpoint_policy.mode == "cooperative"
|
||||
and not payload.checkpoint_policy.allowed_checkpoints
|
||||
) or (
|
||||
payload.checkpoint_policy.mode == "non-checkpointable"
|
||||
and payload.checkpoint_policy.allowed_checkpoints
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim checkpoint policy is inconsistent")
|
||||
|
||||
expected_request_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_REQUEST_SCHEMA,
|
||||
"idempotency_key": payload.idempotency_key,
|
||||
"source_session_id": payload.source.session_id,
|
||||
"source_catalog_sha256": payload.source.catalog_sha256,
|
||||
"source_bundle_sha256": payload.source.bundle_sha256,
|
||||
"source_capability_manifest_sha256": (payload.source.capability_manifest_sha256),
|
||||
"setup_id": payload.setup.setup_id,
|
||||
"definition_sha256": payload.setup.definition_sha256,
|
||||
}
|
||||
)
|
||||
if payload.request_sha256 != expected_request_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job request identity changed")
|
||||
expected_identity_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
"source_session_id": payload.source.session_id,
|
||||
"source_catalog_sha256": payload.source.catalog_sha256,
|
||||
"source_bundle_sha256": payload.source.bundle_sha256,
|
||||
"source_capability_manifest_sha256": (payload.source.capability_manifest_sha256),
|
||||
"setup_id": payload.setup.setup_id,
|
||||
"definition_id": payload.setup.definition_id,
|
||||
"definition_version": payload.setup.definition_version,
|
||||
"definition_sha256": payload.setup.definition_sha256,
|
||||
"source_adapter_id": payload.source.adapter.adapter_id,
|
||||
"source_adapter_version": payload.source.adapter.version,
|
||||
"source_adapter_sha256": payload.source.adapter.adapter_sha256,
|
||||
"executor_release_id": payload.executor.release_id,
|
||||
"executor_release_sha256": payload.executor.release_sha256,
|
||||
"executor_image_sha256": payload.executor.image_sha256,
|
||||
"model_release_ids": payload.executor.model_release_ids,
|
||||
"model_manifest_sha256": payload.executor.model_manifest_sha256,
|
||||
"resource_profile_id": payload.executor.resource_profile_id,
|
||||
"resource_profile_sha256": payload.executor.resource_profile_sha256,
|
||||
"checkpoint_policy": payload.checkpoint_policy.mode,
|
||||
"allowed_checkpoints": payload.checkpoint_policy.allowed_checkpoints,
|
||||
}
|
||||
)
|
||||
if payload.identity_sha256 != expected_identity_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job execution identity changed")
|
||||
expected_submission_receipt_sha256 = _sha256_document(
|
||||
{
|
||||
"schema_version": OBSERVATORY_RECORDED_JOB_SCHEMA,
|
||||
"job_id": payload.job_id,
|
||||
"request_sha256": payload.request_sha256,
|
||||
"identity_sha256": payload.identity_sha256,
|
||||
"state": "accepted",
|
||||
"created_at_utc": payload.created_at_utc,
|
||||
}
|
||||
)
|
||||
if payload.submission_receipt_sha256 != expected_submission_receipt_sha256:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker recorded-job submission receipt changed")
|
||||
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=payload.job_id,
|
||||
request_sha256=payload.request_sha256,
|
||||
identity_sha256=payload.identity_sha256,
|
||||
source_session_id=payload.source.session_id,
|
||||
source_catalog_sha256=payload.source.catalog_sha256,
|
||||
source_bundle_sha256=payload.source.bundle_sha256,
|
||||
source_capability_manifest_sha256=(payload.source.capability_manifest_sha256),
|
||||
source_adapter_id=payload.source.adapter.adapter_id,
|
||||
source_adapter_version=payload.source.adapter.version,
|
||||
source_adapter_sha256=payload.source.adapter.adapter_sha256,
|
||||
setup_id=payload.setup.setup_id,
|
||||
definition_id=payload.setup.definition_id,
|
||||
definition_version=payload.setup.definition_version,
|
||||
definition_sha256=payload.setup.definition_sha256,
|
||||
executor_release_id=payload.executor.release_id,
|
||||
executor_identity=ObservatoryWorkerExecutorIdentity(
|
||||
release_sha256=payload.executor.release_sha256,
|
||||
image_sha256=payload.executor.image_sha256,
|
||||
model_manifest_sha256=payload.executor.model_manifest_sha256,
|
||||
resource_profile_sha256=payload.executor.resource_profile_sha256,
|
||||
),
|
||||
model_release_ids=tuple(payload.executor.model_release_ids),
|
||||
resource_profile_id=payload.executor.resource_profile_id,
|
||||
checkpoint_policy=payload.checkpoint_policy.mode,
|
||||
allowed_checkpoints=tuple(payload.checkpoint_policy.allowed_checkpoints),
|
||||
claim_generation=payload.claim_generation,
|
||||
restart_from_zero=payload.restart_from_zero,
|
||||
)
|
||||
|
||||
|
||||
def _validate_transition_acknowledgement(
|
||||
payload: Mapping[str, object],
|
||||
*,
|
||||
expected_job: SealedObservatoryRecordedJob,
|
||||
expected_state: RecordedJobWireState | tuple[RecordedJobWireState, ...],
|
||||
) -> _RecordedJobPayload:
|
||||
try:
|
||||
acknowledgement = _RecordedJobPayload.model_validate(dict(payload))
|
||||
sealed = _seal_job(acknowledgement)
|
||||
except ObservatoryWorkerClaimRejectedError:
|
||||
raise
|
||||
except (TypeError, ValueError, ValidationError) as exc:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement violates the sealed protocol"
|
||||
) from exc
|
||||
states = (expected_state,) if isinstance(expected_state, str) else expected_state
|
||||
if acknowledgement.state not in states:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement has an unexpected state"
|
||||
)
|
||||
if (
|
||||
sealed.job_id != expected_job.job_id
|
||||
or sealed.identity_sha256 != expected_job.identity_sha256
|
||||
or sealed.claim_generation != expected_job.claim_generation
|
||||
):
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker transition acknowledgement changed job identity"
|
||||
)
|
||||
return acknowledgement
|
||||
|
||||
|
||||
def _default_claim_request_id() -> str:
|
||||
return f"worker-006:{uuid4().hex}"
|
||||
|
||||
|
||||
def _bounded_executor_failure(exc: Exception) -> str:
|
||||
detail = " ".join(str(exc).split())
|
||||
message = f"Executor adapter raised {type(exc).__name__}."
|
||||
if detail:
|
||||
message = f"{message} {detail}"
|
||||
return message[:MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH]
|
||||
|
||||
|
||||
def _sha256_document(document: Mapping[str, object]) -> str:
|
||||
payload = json.dumps(
|
||||
document,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
+99
-25
@@ -47,10 +47,20 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
@@ -160,6 +170,11 @@ from k1link.web.map_api import (
|
||||
)
|
||||
from k1link.web.map_view_api import build_map_view_router
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.observatory_worker_api import (
|
||||
ObservatoryWorkerAuthentication,
|
||||
build_observatory_worker_router,
|
||||
load_observatory_worker_authentication,
|
||||
)
|
||||
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
|
||||
from k1link.web.plugin_runtime import (
|
||||
STATE_READ_ACTION_ID,
|
||||
@@ -245,6 +260,25 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
||||
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
|
||||
|
||||
def _load_optional_observatory_worker_authentication(
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
||||
*,
|
||||
token_path: Path,
|
||||
) -> tuple[ObservatoryWorkerAuthentication | None, str | None]:
|
||||
"""Load the optional Worker credential without widening app startup risk."""
|
||||
|
||||
if recorded_job_queue is None:
|
||||
return None, "Observatory recorded-job queue is unavailable"
|
||||
try:
|
||||
return load_observatory_worker_authentication(token_path), None
|
||||
except ValueError as exc:
|
||||
# Worker pull transport is optional. A missing or unsafe credential
|
||||
# disables only this router; K1, Simulation and legacy LAB still start.
|
||||
return None, str(exc)
|
||||
|
||||
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
||||
(
|
||||
@@ -262,9 +296,7 @@ try:
|
||||
session_store=session_store,
|
||||
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
||||
config=M49QueueBindingConfig.from_file(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "observatory-m49-recorded-queue-binding.json"
|
||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||
),
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||
@@ -279,6 +311,17 @@ except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION = None
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled until claim leases and a verified "
|
||||
"Observatory result publisher are implemented and accepted"
|
||||
)
|
||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
||||
@@ -293,6 +336,39 @@ session_recording_materializer = SessionRecordingMaterializer(
|
||||
session_recorded_media_inspector = RecordedMediaInspector(
|
||||
session_store.data_dir / "recorded-media-preparations"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
try:
|
||||
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
portable_lab_v1_definition = next(
|
||||
definition
|
||||
for definition in portable_definition_registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
portable_source_capability_service = RecordedK1SourceAdmissionService(
|
||||
data_dir=session_store.data_dir,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
|
||||
registry=portable_definition_registry,
|
||||
capability_probe=portable_source_capability_service,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||
except (
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableSetupProjectionError,
|
||||
OSError,
|
||||
StopIteration,
|
||||
ValueError,
|
||||
) as exc:
|
||||
# Portable LAB V1 is an optional observation-only slice. A drifted
|
||||
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
||||
_ffmpeg = _resolve_media_tool("ffmpeg")
|
||||
_ffprobe = _resolve_media_tool("ffprobe")
|
||||
session_recorded_camera_frame_service = (
|
||||
@@ -765,8 +841,23 @@ app.include_router(
|
||||
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
||||
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||
)
|
||||
)
|
||||
if (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
):
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment")
|
||||
)
|
||||
@@ -1066,10 +1157,7 @@ app.include_router(
|
||||
spatial_evidence_provider=m48_raw_evidence_reader,
|
||||
evaluation_runner=LABORATORY_RUNNER,
|
||||
evaluation_receipt_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "laboratory-run-receipts"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts"
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -1093,33 +1181,21 @@ app.include_router(
|
||||
app.include_router(
|
||||
build_m49_tgs_fail_closed_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-fail-closed-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m49_tgs_full_shadow_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-full-shadow-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_vegetation_shadow_lab_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "lab-v1-vegetation"
|
||||
/ "results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results"
|
||||
),
|
||||
canonical_recording_provider=_canonical_lab_recording_source,
|
||||
camera_frame_provider=(
|
||||
@@ -1128,9 +1204,7 @@ app.include_router(
|
||||
else None
|
||||
),
|
||||
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
||||
rerun_overlay_cache_root=(
|
||||
session_store.data_dir / "laboratory-rerun-overlays"
|
||||
),
|
||||
rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"),
|
||||
ffmpeg_path=_ffmpeg,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -22,6 +22,10 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
@@ -32,24 +36,24 @@ from k1link.observatory.recorded_jobs import (
|
||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal[
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal["missioncore.observatory-lab-projection/v1"] = (
|
||||
"missioncore.observatory-lab-projection/v1"
|
||||
] = "missioncore.observatory-lab-projection/v1"
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal[
|
||||
)
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal["missioncore.observatory-lab-projection-rename/v1"] = (
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
] = "missioncore.observatory-lab-projection-rename/v1"
|
||||
)
|
||||
OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
|
||||
"missioncore.observatory-run-preflight-request/v1"
|
||||
] = "missioncore.observatory-run-preflight-request/v1"
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal["missioncore.observatory-run-preflight/v1"] = (
|
||||
"missioncore.observatory-run-preflight/v1"
|
||||
] = "missioncore.observatory-run-preflight/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
|
||||
"missioncore.observatory-recorded-run-submit/v1"
|
||||
] = "missioncore.observatory-recorded-run-submit/v1"
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal["missioncore.observatory-recorded-job-list/v1"] = (
|
||||
"missioncore.observatory-recorded-job-list/v1"
|
||||
] = "missioncore.observatory-recorded-job-list/v1"
|
||||
)
|
||||
|
||||
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
|
||||
"commands_enabled": False,
|
||||
@@ -64,9 +68,7 @@ class _StrictApiModel(BaseModel):
|
||||
|
||||
|
||||
class ObservatoryProjectionRenameRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-lab-projection-rename/v1"]
|
||||
display_name: str = Field(min_length=1, max_length=160)
|
||||
|
||||
|
||||
@@ -92,9 +94,7 @@ class ObservatoryRunPreflightRequest(_StrictApiModel):
|
||||
|
||||
|
||||
class ObservatoryRunPreparationRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-run-preparation-request/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-run-preparation-request/v1"]
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
@@ -142,6 +142,8 @@ def build_observatory_router(
|
||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||
recorded_job_queue_error: str | None = None,
|
||||
portable_setup_projector: PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector_error: str | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
@@ -212,6 +214,41 @@ def build_observatory_router(
|
||||
available.add(result_id)
|
||||
return frozenset(available)
|
||||
|
||||
if portable_setup_projector is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def list_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
source = source_summary(source_session_id)
|
||||
try:
|
||||
return portable_setup_projector.catalog(source)
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
|
||||
) from exc
|
||||
|
||||
elif portable_setup_projector_error is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def unavailable_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> None:
|
||||
del source_session_id
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 недоступен.",
|
||||
)
|
||||
|
||||
if setup_registry is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/laboratory-setups")
|
||||
@@ -225,9 +262,7 @@ def build_observatory_router(
|
||||
source = source_summary(source_session_id)
|
||||
return setup_registry.catalog(
|
||||
source,
|
||||
available_observatory_result_ids=available_observatory_results(
|
||||
source_session_id
|
||||
),
|
||||
available_observatory_result_ids=available_observatory_results(source_session_id),
|
||||
)
|
||||
|
||||
@router.post("/api/v1/observatory/run-preflights")
|
||||
@@ -254,9 +289,7 @@ def build_observatory_router(
|
||||
)
|
||||
definition = projected["run_definition"]
|
||||
expected_digest = (
|
||||
definition.get("definition_sha256")
|
||||
if isinstance(definition, dict)
|
||||
else None
|
||||
definition.get("definition_sha256") if isinstance(definition, dict) else None
|
||||
)
|
||||
if request.definition_sha256 != expected_digest:
|
||||
raise HTTPException(
|
||||
@@ -276,8 +309,7 @@ def build_observatory_router(
|
||||
binding_service = recorded_binding_service
|
||||
exact_queue_setup = (
|
||||
binding_service is not None
|
||||
and request.setup_id
|
||||
== binding_service.config.setup.setup_id
|
||||
and request.setup_id == binding_service.config.setup.setup_id
|
||||
)
|
||||
if (
|
||||
compatible
|
||||
@@ -303,9 +335,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "source-compatibility",
|
||||
"outcome": "pass" if compatible else "fail",
|
||||
"reason_code": (
|
||||
"source-compatible" if compatible else "source-incompatible"
|
||||
),
|
||||
"reason_code": ("source-compatible" if compatible else "source-incompatible"),
|
||||
"message": (
|
||||
"Источник точно совместим с сохранённым сетапом."
|
||||
if compatible
|
||||
@@ -323,11 +353,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "executor",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queue_binding_ready
|
||||
else "fail"
|
||||
"not-applicable" if existing else "pass" if queue_binding_ready else "fail"
|
||||
),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-executor"
|
||||
@@ -349,13 +375,7 @@ def build_observatory_router(
|
||||
},
|
||||
{
|
||||
"check_id": "durable-queue",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queueable
|
||||
else "fail"
|
||||
),
|
||||
"outcome": ("not-applicable" if existing else "pass" if queueable else "fail"),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-queue"
|
||||
if existing
|
||||
@@ -378,9 +398,7 @@ def build_observatory_router(
|
||||
"source_session_id": request.source_session_id,
|
||||
"setup_id": request.setup_id,
|
||||
"definition_sha256": expected_digest,
|
||||
"outcome": (
|
||||
"existing" if existing else "queueable" if queueable else "blocked"
|
||||
),
|
||||
"outcome": ("existing" if existing else "queueable" if queueable else "blocked"),
|
||||
"submission_allowed": queueable,
|
||||
"checks": checks,
|
||||
"existing_result_ids": preflight.get("existing_result_ids", []),
|
||||
@@ -421,9 +439,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRunPreparationRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(request.idempotency_key)
|
||||
request_sha256 = observatory_run_preparation_request_sha256(
|
||||
idempotency_key=request.idempotency_key,
|
||||
source_session_id=request.source_session_id,
|
||||
@@ -634,9 +650,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
existing_job = None
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Authenticated pull transport for the durable Observatory recorded-job queue.
|
||||
|
||||
The transport is deliberately narrower than an execution API. Worker callers
|
||||
can claim a server-sealed RunDefinition and advance its durable state, but they
|
||||
cannot supply commands, paths, environment variables, container images, or
|
||||
priority. Those execution identities remain part of the queue-owned job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedCheckpointError,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedPreemptionError,
|
||||
ObservatoryRecordedQueueBusyError,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueIntegrityError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
)
|
||||
|
||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-checkpoint-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-succeed-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
||||
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_JOB_ID_PATTERN = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN = r"^[a-f0-9]{64}$"
|
||||
_CLAIM_REQUEST_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
|
||||
_SESSION_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
_IDENTIFIER_PATTERN = r"^[a-z][a-z0-9-]{2,95}$"
|
||||
_WORKER_BEARER = HTTPBearer(auto_error=False)
|
||||
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerAuthentication:
|
||||
"""Server-owned Worker identity and the SHA-256 of its bearer secret."""
|
||||
|
||||
bearer_token_sha256: str
|
||||
contour_id: str = "worker-006"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _SHA256.fullmatch(self.bearer_token_sha256) is None:
|
||||
raise ValueError("Worker bearer token SHA-256 is invalid")
|
||||
if _IDENTIFIER.fullmatch(self.contour_id) is None:
|
||||
raise ValueError("Worker contour id is invalid")
|
||||
|
||||
|
||||
def load_observatory_worker_authentication(
|
||||
token_path: Path,
|
||||
*,
|
||||
contour_id: str = "worker-006",
|
||||
) -> ObservatoryWorkerAuthentication:
|
||||
"""Load one local Worker credential without retaining its plaintext.
|
||||
|
||||
The credential file is an operator/deployment concern. Mission Core only
|
||||
retains its SHA-256 in the router configuration and refuses symlinks,
|
||||
non-regular files, or group/other permissions.
|
||||
"""
|
||||
|
||||
candidate = token_path.expanduser().absolute()
|
||||
descriptor: int | None = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
candidate,
|
||||
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ValueError("Worker bearer credential must be a regular file")
|
||||
if metadata.st_mode & 0o077:
|
||||
raise ValueError("Worker bearer credential permissions are too broad")
|
||||
if not 32 <= metadata.st_size <= 512:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
descriptor = None
|
||||
payload = stream.read(513)
|
||||
except ValueError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ValueError("Worker bearer credential is unavailable") from exc
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
token = payload.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("Worker bearer credential is not ASCII") from exc
|
||||
if _TOKEN.fullmatch(token) is None:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
return ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
contour_id=contour_id,
|
||||
)
|
||||
|
||||
|
||||
class _StrictWorkerRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-claim-request/v1"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-start-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
checkpoint_id: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerSucceedRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-succeed-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
result_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=_SESSION_ID_PATTERN,
|
||||
)
|
||||
result_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class ObservatoryWorkerFailRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-fail-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
error_code: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
def build_observatory_worker_router(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
authentication: ObservatoryWorkerAuthentication,
|
||||
) -> APIRouter:
|
||||
"""Build the bounded Worker pull/state-transition router.
|
||||
|
||||
``authentication`` contains only a token digest. The plaintext bearer
|
||||
secret exists transiently while FastAPI parses one request, is immediately
|
||||
hashed, and is compared to the configured digest in constant time.
|
||||
"""
|
||||
|
||||
def require_configured_worker(
|
||||
credentials: Annotated[
|
||||
HTTPAuthorizationCredentials | None,
|
||||
Depends(_WORKER_BEARER),
|
||||
],
|
||||
contour_id: Annotated[
|
||||
str | None,
|
||||
Header(alias=OBSERVATORY_WORKER_CONTOUR_HEADER),
|
||||
] = None,
|
||||
) -> None:
|
||||
if credentials is None or credentials.scheme.lower() != "bearer":
|
||||
raise _unauthorized()
|
||||
token = credentials.credentials
|
||||
if not token or len(token) > 512:
|
||||
raise _unauthorized()
|
||||
supplied_sha256 = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
if not hmac.compare_digest(
|
||||
supplied_sha256,
|
||||
authentication.bearer_token_sha256,
|
||||
):
|
||||
raise _unauthorized()
|
||||
if contour_id is None or not hmac.compare_digest(
|
||||
contour_id,
|
||||
authentication.contour_id,
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Worker contour identity was rejected.",
|
||||
)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/worker/observatory",
|
||||
tags=["observatory-worker"],
|
||||
dependencies=[Depends(require_configured_worker)],
|
||||
)
|
||||
|
||||
@router.post("/recorded-jobs/claims", response_model=None)
|
||||
def claim_next(
|
||||
request: ObservatoryWorkerClaimRequest,
|
||||
) -> dict[str, object] | Response:
|
||||
claim = _queue_call(
|
||||
lambda: queue.claim_next(
|
||||
claimant_id=authentication.contour_id,
|
||||
claim_request_id=request.claim_request_id,
|
||||
)
|
||||
)
|
||||
if claim is None:
|
||||
return Response(status_code=204)
|
||||
return claim.as_dict()
|
||||
|
||||
@router.get("/recorded-jobs/{job_id}")
|
||||
def get_job(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.get(job_id)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/start")
|
||||
def start_job(
|
||||
request: ObservatoryWorkerStartRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.checkpoint(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
checkpoint_id=request.checkpoint_id,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/succeed")
|
||||
def succeed_job(
|
||||
request: ObservatoryWorkerSucceedRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/fail")
|
||||
def fail_job(
|
||||
request: ObservatoryWorkerFailRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.fail(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
error_code=request.error_code,
|
||||
message=request.message,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _unauthorized() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=401,
|
||||
detail="Worker bearer credential was rejected.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def _queue_call[T](operation: Callable[[], T]) -> T:
|
||||
try:
|
||||
return operation()
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Recorded job was not found.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueStaleClaimError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job claim is stale.",
|
||||
) from exc
|
||||
except ObservatoryRecordedCheckpointError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job checkpoint was rejected.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job transition conflicts with durable state.",
|
||||
) from exc
|
||||
except (ObservatoryRecordedQueueBusyError, ObservatoryRecordedPreemptionError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job resources are reserved for live K1 work.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueCapacityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue capacity is unavailable.",
|
||||
headers={"Retry-After": "5"},
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueIntegrityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue integrity is unavailable.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue is unavailable.",
|
||||
) from exc
|
||||
Reference in New Issue
Block a user