feat(observatory): add portable LAB V1 foundation
This commit is contained in:
+99
-25
@@ -47,10 +47,20 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinitionRegistry,
|
||||
PortableRunDefinitionRegistryError,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PORTABLE_LAB_V1_SETUP_ID,
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
@@ -160,6 +170,11 @@ from k1link.web.map_api import (
|
||||
)
|
||||
from k1link.web.map_view_api import build_map_view_router
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.observatory_worker_api import (
|
||||
ObservatoryWorkerAuthentication,
|
||||
build_observatory_worker_router,
|
||||
load_observatory_worker_authentication,
|
||||
)
|
||||
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
|
||||
from k1link.web.plugin_runtime import (
|
||||
STATE_READ_ACTION_ID,
|
||||
@@ -245,6 +260,25 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
||||
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
|
||||
|
||||
def _load_optional_observatory_worker_authentication(
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None,
|
||||
*,
|
||||
token_path: Path,
|
||||
) -> tuple[ObservatoryWorkerAuthentication | None, str | None]:
|
||||
"""Load the optional Worker credential without widening app startup risk."""
|
||||
|
||||
if recorded_job_queue is None:
|
||||
return None, "Observatory recorded-job queue is unavailable"
|
||||
try:
|
||||
return load_observatory_worker_authentication(token_path), None
|
||||
except ValueError as exc:
|
||||
# Worker pull transport is optional. A missing or unsafe credential
|
||||
# disables only this router; K1, Simulation and legacy LAB still start.
|
||||
return None, str(exc)
|
||||
|
||||
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None
|
||||
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
|
||||
(
|
||||
@@ -262,9 +296,7 @@ try:
|
||||
session_store=session_store,
|
||||
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
|
||||
config=M49QueueBindingConfig.from_file(
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "observatory-m49-recorded-queue-binding.json"
|
||||
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
|
||||
),
|
||||
)
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
|
||||
@@ -279,6 +311,17 @@ except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError
|
||||
OBSERVATORY_RECORDED_BINDING_SERVICE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE = None
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
|
||||
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
|
||||
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
|
||||
OBSERVATORY_WORKER_API_ERROR: str | None
|
||||
OBSERVATORY_WORKER_AUTHENTICATION = None
|
||||
OBSERVATORY_WORKER_API_ERROR = (
|
||||
"Worker pull API is hard-disabled until claim leases and a verified "
|
||||
"Observatory result publisher are implemented and accepted"
|
||||
)
|
||||
simulation_project_store = SimulationProjectStore(session_store.data_dir)
|
||||
simulation_project_service = SimulationProjectService(simulation_project_store)
|
||||
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
|
||||
@@ -293,6 +336,39 @@ session_recording_materializer = SessionRecordingMaterializer(
|
||||
session_recorded_media_inspector = RecordedMediaInspector(
|
||||
session_store.data_dir / "recorded-media-preparations"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
try:
|
||||
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
|
||||
)
|
||||
portable_lab_v1_definition = next(
|
||||
definition
|
||||
for definition in portable_definition_registry.definitions
|
||||
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
|
||||
)
|
||||
portable_source_capability_service = RecordedK1SourceAdmissionService(
|
||||
data_dir=session_store.data_dir,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
|
||||
registry=portable_definition_registry,
|
||||
capability_probe=portable_source_capability_service,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
|
||||
except (
|
||||
PortableRunDefinitionRegistryError,
|
||||
PortableSetupProjectionError,
|
||||
OSError,
|
||||
StopIteration,
|
||||
ValueError,
|
||||
) as exc:
|
||||
# Portable LAB V1 is an optional observation-only slice. A drifted
|
||||
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
|
||||
_ffmpeg = _resolve_media_tool("ffmpeg")
|
||||
_ffprobe = _resolve_media_tool("ffprobe")
|
||||
session_recorded_camera_frame_service = (
|
||||
@@ -765,8 +841,23 @@ app.include_router(
|
||||
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
|
||||
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
|
||||
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
|
||||
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
|
||||
)
|
||||
)
|
||||
if (
|
||||
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
|
||||
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
and OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
):
|
||||
app.include_router(
|
||||
build_observatory_worker_router(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment")
|
||||
)
|
||||
@@ -1066,10 +1157,7 @@ app.include_router(
|
||||
spatial_evidence_provider=m48_raw_evidence_reader,
|
||||
evaluation_runner=LABORATORY_RUNNER,
|
||||
evaluation_receipt_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "laboratory-run-receipts"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts"
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -1093,33 +1181,21 @@ app.include_router(
|
||||
app.include_router(
|
||||
build_m49_tgs_fail_closed_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-fail-closed-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m49_tgs_full_shadow_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m49"
|
||||
/ "tgs-full-shadow-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_vegetation_shadow_lab_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "lab-v1-vegetation"
|
||||
/ "results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results"
|
||||
),
|
||||
canonical_recording_provider=_canonical_lab_recording_source,
|
||||
camera_frame_provider=(
|
||||
@@ -1128,9 +1204,7 @@ app.include_router(
|
||||
else None
|
||||
),
|
||||
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
||||
rerun_overlay_cache_root=(
|
||||
session_store.data_dir / "laboratory-rerun-overlays"
|
||||
),
|
||||
rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"),
|
||||
ffmpeg_path=_ffmpeg,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -22,6 +22,10 @@ from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingIntegrityError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_setup_projection import (
|
||||
PortableLabV1SetupProjector,
|
||||
PortableSetupProjectionError,
|
||||
)
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
@@ -32,24 +36,24 @@ from k1link.observatory.recorded_jobs import (
|
||||
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
|
||||
from k1link.sessions.models import SessionSummary
|
||||
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal[
|
||||
OBSERVATORY_PROJECTION_SCHEMA: Literal["missioncore.observatory-lab-projection/v1"] = (
|
||||
"missioncore.observatory-lab-projection/v1"
|
||||
] = "missioncore.observatory-lab-projection/v1"
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal[
|
||||
)
|
||||
OBSERVATORY_RENAME_SCHEMA: Literal["missioncore.observatory-lab-projection-rename/v1"] = (
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
] = "missioncore.observatory-lab-projection-rename/v1"
|
||||
)
|
||||
OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
|
||||
"missioncore.observatory-run-preflight-request/v1"
|
||||
] = "missioncore.observatory-run-preflight-request/v1"
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
|
||||
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal["missioncore.observatory-run-preflight/v1"] = (
|
||||
"missioncore.observatory-run-preflight/v1"
|
||||
] = "missioncore.observatory-run-preflight/v1"
|
||||
)
|
||||
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
|
||||
"missioncore.observatory-recorded-run-submit/v1"
|
||||
] = "missioncore.observatory-recorded-run-submit/v1"
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
|
||||
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal["missioncore.observatory-recorded-job-list/v1"] = (
|
||||
"missioncore.observatory-recorded-job-list/v1"
|
||||
] = "missioncore.observatory-recorded-job-list/v1"
|
||||
)
|
||||
|
||||
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
|
||||
"commands_enabled": False,
|
||||
@@ -64,9 +68,7 @@ class _StrictApiModel(BaseModel):
|
||||
|
||||
|
||||
class ObservatoryProjectionRenameRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-lab-projection-rename/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-lab-projection-rename/v1"]
|
||||
display_name: str = Field(min_length=1, max_length=160)
|
||||
|
||||
|
||||
@@ -92,9 +94,7 @@ class ObservatoryRunPreflightRequest(_StrictApiModel):
|
||||
|
||||
|
||||
class ObservatoryRunPreparationRequest(_StrictApiModel):
|
||||
schema_version: Literal[
|
||||
"missioncore.observatory-run-preparation-request/v1"
|
||||
]
|
||||
schema_version: Literal["missioncore.observatory-run-preparation-request/v1"]
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
@@ -142,6 +142,8 @@ def build_observatory_router(
|
||||
recorded_binding_service: M49RecordedQueueBindingService | None = None,
|
||||
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
|
||||
recorded_job_queue_error: str | None = None,
|
||||
portable_setup_projector: PortableLabV1SetupProjector | None = None,
|
||||
portable_setup_projector_error: str | None = None,
|
||||
) -> APIRouter:
|
||||
"""Build bounded catalog-only mutations for typed Observatory projections."""
|
||||
|
||||
@@ -212,6 +214,41 @@ def build_observatory_router(
|
||||
available.add(result_id)
|
||||
return frozenset(available)
|
||||
|
||||
if portable_setup_projector is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def list_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
source = source_summary(source_session_id)
|
||||
try:
|
||||
return portable_setup_projector.catalog(source)
|
||||
except PortableSetupProjectionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
|
||||
) from exc
|
||||
|
||||
elif portable_setup_projector_error is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/portable-laboratory-setups")
|
||||
def unavailable_observatory_portable_laboratory_setups(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> None:
|
||||
del source_session_id
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Portable-каталог LAB V1 недоступен.",
|
||||
)
|
||||
|
||||
if setup_registry is not None:
|
||||
|
||||
@router.get("/api/v1/observatory/laboratory-setups")
|
||||
@@ -225,9 +262,7 @@ def build_observatory_router(
|
||||
source = source_summary(source_session_id)
|
||||
return setup_registry.catalog(
|
||||
source,
|
||||
available_observatory_result_ids=available_observatory_results(
|
||||
source_session_id
|
||||
),
|
||||
available_observatory_result_ids=available_observatory_results(source_session_id),
|
||||
)
|
||||
|
||||
@router.post("/api/v1/observatory/run-preflights")
|
||||
@@ -254,9 +289,7 @@ def build_observatory_router(
|
||||
)
|
||||
definition = projected["run_definition"]
|
||||
expected_digest = (
|
||||
definition.get("definition_sha256")
|
||||
if isinstance(definition, dict)
|
||||
else None
|
||||
definition.get("definition_sha256") if isinstance(definition, dict) else None
|
||||
)
|
||||
if request.definition_sha256 != expected_digest:
|
||||
raise HTTPException(
|
||||
@@ -276,8 +309,7 @@ def build_observatory_router(
|
||||
binding_service = recorded_binding_service
|
||||
exact_queue_setup = (
|
||||
binding_service is not None
|
||||
and request.setup_id
|
||||
== binding_service.config.setup.setup_id
|
||||
and request.setup_id == binding_service.config.setup.setup_id
|
||||
)
|
||||
if (
|
||||
compatible
|
||||
@@ -303,9 +335,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "source-compatibility",
|
||||
"outcome": "pass" if compatible else "fail",
|
||||
"reason_code": (
|
||||
"source-compatible" if compatible else "source-incompatible"
|
||||
),
|
||||
"reason_code": ("source-compatible" if compatible else "source-incompatible"),
|
||||
"message": (
|
||||
"Источник точно совместим с сохранённым сетапом."
|
||||
if compatible
|
||||
@@ -323,11 +353,7 @@ def build_observatory_router(
|
||||
{
|
||||
"check_id": "executor",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queue_binding_ready
|
||||
else "fail"
|
||||
"not-applicable" if existing else "pass" if queue_binding_ready else "fail"
|
||||
),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-executor"
|
||||
@@ -349,13 +375,7 @@ def build_observatory_router(
|
||||
},
|
||||
{
|
||||
"check_id": "durable-queue",
|
||||
"outcome": (
|
||||
"not-applicable"
|
||||
if existing
|
||||
else "pass"
|
||||
if queueable
|
||||
else "fail"
|
||||
),
|
||||
"outcome": ("not-applicable" if existing else "pass" if queueable else "fail"),
|
||||
"reason_code": (
|
||||
"existing-result-does-not-require-queue"
|
||||
if existing
|
||||
@@ -378,9 +398,7 @@ def build_observatory_router(
|
||||
"source_session_id": request.source_session_id,
|
||||
"setup_id": request.setup_id,
|
||||
"definition_sha256": expected_digest,
|
||||
"outcome": (
|
||||
"existing" if existing else "queueable" if queueable else "blocked"
|
||||
),
|
||||
"outcome": ("existing" if existing else "queueable" if queueable else "blocked"),
|
||||
"submission_allowed": queueable,
|
||||
"checks": checks,
|
||||
"existing_result_ids": preflight.get("existing_result_ids", []),
|
||||
@@ -421,9 +439,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRunPreparationRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing = run_preparation_ledger.get_by_idempotency_key(request.idempotency_key)
|
||||
request_sha256 = observatory_run_preparation_request_sha256(
|
||||
idempotency_key=request.idempotency_key,
|
||||
source_session_id=request.source_session_id,
|
||||
@@ -634,9 +650,7 @@ def build_observatory_router(
|
||||
request: ObservatoryRecordedRunSubmitRequest,
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(
|
||||
request.idempotency_key
|
||||
)
|
||||
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
|
||||
except ObservatoryRecordedQueueNotFoundError:
|
||||
existing_job = None
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Authenticated pull transport for the durable Observatory recorded-job queue.
|
||||
|
||||
The transport is deliberately narrower than an execution API. Worker callers
|
||||
can claim a server-sealed RunDefinition and advance its durable state, but they
|
||||
cannot supply commands, paths, environment variables, container images, or
|
||||
priority. Those execution identities remain part of the queue-owned job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Response
|
||||
from fastapi import Path as ApiPath
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedCheckpointError,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedPreemptionError,
|
||||
ObservatoryRecordedQueueBusyError,
|
||||
ObservatoryRecordedQueueCapacityError,
|
||||
ObservatoryRecordedQueueConflictError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryRecordedQueueIntegrityError,
|
||||
ObservatoryRecordedQueueNotFoundError,
|
||||
ObservatoryRecordedQueueStaleClaimError,
|
||||
)
|
||||
|
||||
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
|
||||
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
|
||||
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-checkpoint-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
|
||||
"missioncore.observatory-worker-succeed-request/v1"
|
||||
)
|
||||
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
|
||||
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
|
||||
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_JOB_ID_PATTERN = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN = r"^[a-f0-9]{64}$"
|
||||
_CLAIM_REQUEST_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
|
||||
_SESSION_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
_IDENTIFIER_PATTERN = r"^[a-z][a-z0-9-]{2,95}$"
|
||||
_WORKER_BEARER = HTTPBearer(auto_error=False)
|
||||
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservatoryWorkerAuthentication:
|
||||
"""Server-owned Worker identity and the SHA-256 of its bearer secret."""
|
||||
|
||||
bearer_token_sha256: str
|
||||
contour_id: str = "worker-006"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _SHA256.fullmatch(self.bearer_token_sha256) is None:
|
||||
raise ValueError("Worker bearer token SHA-256 is invalid")
|
||||
if _IDENTIFIER.fullmatch(self.contour_id) is None:
|
||||
raise ValueError("Worker contour id is invalid")
|
||||
|
||||
|
||||
def load_observatory_worker_authentication(
|
||||
token_path: Path,
|
||||
*,
|
||||
contour_id: str = "worker-006",
|
||||
) -> ObservatoryWorkerAuthentication:
|
||||
"""Load one local Worker credential without retaining its plaintext.
|
||||
|
||||
The credential file is an operator/deployment concern. Mission Core only
|
||||
retains its SHA-256 in the router configuration and refuses symlinks,
|
||||
non-regular files, or group/other permissions.
|
||||
"""
|
||||
|
||||
candidate = token_path.expanduser().absolute()
|
||||
descriptor: int | None = None
|
||||
try:
|
||||
descriptor = os.open(
|
||||
candidate,
|
||||
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
|
||||
)
|
||||
metadata = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ValueError("Worker bearer credential must be a regular file")
|
||||
if metadata.st_mode & 0o077:
|
||||
raise ValueError("Worker bearer credential permissions are too broad")
|
||||
if not 32 <= metadata.st_size <= 512:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
descriptor = None
|
||||
payload = stream.read(513)
|
||||
except ValueError:
|
||||
raise
|
||||
except OSError as exc:
|
||||
raise ValueError("Worker bearer credential is unavailable") from exc
|
||||
finally:
|
||||
if descriptor is not None:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
token = payload.decode("ascii")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError("Worker bearer credential is not ASCII") from exc
|
||||
if _TOKEN.fullmatch(token) is None:
|
||||
raise ValueError("Worker bearer credential format is invalid")
|
||||
return ObservatoryWorkerAuthentication(
|
||||
bearer_token_sha256=hashlib.sha256(payload).hexdigest(),
|
||||
contour_id=contour_id,
|
||||
)
|
||||
|
||||
|
||||
class _StrictWorkerRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-claim-request/v1"]
|
||||
claim_request_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=_CLAIM_REQUEST_ID_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-start-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
|
||||
|
||||
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
checkpoint_id: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
|
||||
|
||||
class ObservatoryWorkerSucceedRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-succeed-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
result_id: str = Field(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=_SESSION_ID_PATTERN,
|
||||
)
|
||||
result_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class ObservatoryWorkerFailRequest(_StrictWorkerRequest):
|
||||
schema_version: Literal["missioncore.observatory-worker-fail-request/v1"]
|
||||
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
|
||||
error_code: str = Field(
|
||||
min_length=3,
|
||||
max_length=96,
|
||||
pattern=_IDENTIFIER_PATTERN,
|
||||
)
|
||||
message: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
|
||||
def build_observatory_worker_router(
|
||||
queue: ObservatoryRecordedJobQueue,
|
||||
*,
|
||||
authentication: ObservatoryWorkerAuthentication,
|
||||
) -> APIRouter:
|
||||
"""Build the bounded Worker pull/state-transition router.
|
||||
|
||||
``authentication`` contains only a token digest. The plaintext bearer
|
||||
secret exists transiently while FastAPI parses one request, is immediately
|
||||
hashed, and is compared to the configured digest in constant time.
|
||||
"""
|
||||
|
||||
def require_configured_worker(
|
||||
credentials: Annotated[
|
||||
HTTPAuthorizationCredentials | None,
|
||||
Depends(_WORKER_BEARER),
|
||||
],
|
||||
contour_id: Annotated[
|
||||
str | None,
|
||||
Header(alias=OBSERVATORY_WORKER_CONTOUR_HEADER),
|
||||
] = None,
|
||||
) -> None:
|
||||
if credentials is None or credentials.scheme.lower() != "bearer":
|
||||
raise _unauthorized()
|
||||
token = credentials.credentials
|
||||
if not token or len(token) > 512:
|
||||
raise _unauthorized()
|
||||
supplied_sha256 = hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
if not hmac.compare_digest(
|
||||
supplied_sha256,
|
||||
authentication.bearer_token_sha256,
|
||||
):
|
||||
raise _unauthorized()
|
||||
if contour_id is None or not hmac.compare_digest(
|
||||
contour_id,
|
||||
authentication.contour_id,
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Worker contour identity was rejected.",
|
||||
)
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/worker/observatory",
|
||||
tags=["observatory-worker"],
|
||||
dependencies=[Depends(require_configured_worker)],
|
||||
)
|
||||
|
||||
@router.post("/recorded-jobs/claims", response_model=None)
|
||||
def claim_next(
|
||||
request: ObservatoryWorkerClaimRequest,
|
||||
) -> dict[str, object] | Response:
|
||||
claim = _queue_call(
|
||||
lambda: queue.claim_next(
|
||||
claimant_id=authentication.contour_id,
|
||||
claim_request_id=request.claim_request_id,
|
||||
)
|
||||
)
|
||||
if claim is None:
|
||||
return Response(status_code=204)
|
||||
return claim.as_dict()
|
||||
|
||||
@router.get("/recorded-jobs/{job_id}")
|
||||
def get_job(
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.get(job_id)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/start")
|
||||
def start_job(
|
||||
request: ObservatoryWorkerStartRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/checkpoint")
|
||||
def checkpoint_job(
|
||||
request: ObservatoryWorkerCheckpointRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.checkpoint(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
checkpoint_id=request.checkpoint_id,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/succeed")
|
||||
def succeed_job(
|
||||
request: ObservatoryWorkerSucceedRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.succeed(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
result_id=request.result_id,
|
||||
result_sha256=request.result_sha256,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
@router.post("/recorded-jobs/{job_id}/fail")
|
||||
def fail_job(
|
||||
request: ObservatoryWorkerFailRequest,
|
||||
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
|
||||
) -> dict[str, object]:
|
||||
return _queue_call(
|
||||
lambda: queue.fail(
|
||||
job_id,
|
||||
claim_token=request.claim_token,
|
||||
error_code=request.error_code,
|
||||
message=request.message,
|
||||
)
|
||||
).as_dict()
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _unauthorized() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=401,
|
||||
detail="Worker bearer credential was rejected.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def _queue_call[T](operation: Callable[[], T]) -> T:
|
||||
try:
|
||||
return operation()
|
||||
except ObservatoryRecordedQueueNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Recorded job was not found.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueStaleClaimError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job claim is stale.",
|
||||
) from exc
|
||||
except ObservatoryRecordedCheckpointError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job checkpoint was rejected.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job transition conflicts with durable state.",
|
||||
) from exc
|
||||
except (ObservatoryRecordedQueueBusyError, ObservatoryRecordedPreemptionError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Recorded-job resources are reserved for live K1 work.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueCapacityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue capacity is unavailable.",
|
||||
headers={"Retry-After": "5"},
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueIntegrityError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue integrity is unavailable.",
|
||||
) from exc
|
||||
except ObservatoryRecordedQueueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Recorded-job queue is unavailable.",
|
||||
) from exc
|
||||
Reference in New Issue
Block a user