feat(observatory): add portable LAB V1 foundation

This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 11:02:34 +03:00
parent d67e86176e
commit c9302c78c2
27 changed files with 7629 additions and 266 deletions
+99 -25
View File
@@ -47,10 +47,20 @@ from k1link.observatory.m49_queue_binding import (
M49QueueBindingError,
M49RecordedQueueBindingService,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
PortableRunDefinitionRegistryError,
)
from k1link.observatory.portable_setup_projection import (
PORTABLE_LAB_V1_SETUP_ID,
PortableLabV1SetupProjector,
PortableSetupProjectionError,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueError,
)
from k1link.observatory.source_admission import RecordedK1SourceAdmissionService
from k1link.sessions import (
MaterializedRecording,
RecordedCameraFrameService,
@@ -160,6 +170,11 @@ from k1link.web.map_api import (
)
from k1link.web.map_view_api import build_map_view_router
from k1link.web.observatory_api import build_observatory_router
from k1link.web.observatory_worker_api import (
ObservatoryWorkerAuthentication,
build_observatory_worker_router,
load_observatory_worker_authentication,
)
from k1link.web.plugin_catalog import DevicePluginCatalog, PluginCatalogError
from k1link.web.plugin_runtime import (
STATE_READ_ACTION_ID,
@@ -245,6 +260,25 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
session_store = SessionStore(REPOSITORY_ROOT)
def _load_optional_observatory_worker_authentication(
recorded_job_queue: ObservatoryRecordedJobQueue | None,
*,
token_path: Path,
) -> tuple[ObservatoryWorkerAuthentication | None, str | None]:
"""Load the optional Worker credential without widening app startup risk."""
if recorded_job_queue is None:
return None, "Observatory recorded-job queue is unavailable"
try:
return load_observatory_worker_authentication(token_path), None
except ValueError as exc:
# Worker pull transport is optional. A missing or unsafe credential
# disables only this router; K1, Simulation and legacy LAB still start.
return None, str(exc)
OBSERVATORY_RUN_PREPARATION_LEDGER: ObservatoryRunPreparationLedger | None
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
(
@@ -262,9 +296,7 @@ try:
session_store=session_store,
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
config=M49QueueBindingConfig.from_file(
REPOSITORY_ROOT
/ "config"
/ "observatory-m49-recorded-queue-binding.json"
REPOSITORY_ROOT / "config" / "observatory-m49-recorded-queue-binding.json"
),
)
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
@@ -279,6 +311,17 @@ except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError
OBSERVATORY_RECORDED_BINDING_SERVICE = None
OBSERVATORY_RECORDED_JOB_QUEUE = None
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
OBSERVATORY_WORKER_TOKEN_PATH = session_store.data_dir / "worker-auth" / "observatory-worker.token"
OBSERVATORY_WORKER_CLAIM_LEASE_READY = False
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = False
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED = False
OBSERVATORY_WORKER_AUTHENTICATION: ObservatoryWorkerAuthentication | None
OBSERVATORY_WORKER_API_ERROR: str | None
OBSERVATORY_WORKER_AUTHENTICATION = None
OBSERVATORY_WORKER_API_ERROR = (
"Worker pull API is hard-disabled until claim leases and a verified "
"Observatory result publisher are implemented and accepted"
)
simulation_project_store = SimulationProjectStore(session_store.data_dir)
simulation_project_service = SimulationProjectService(simulation_project_store)
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
@@ -293,6 +336,39 @@ session_recording_materializer = SessionRecordingMaterializer(
session_recorded_media_inspector = RecordedMediaInspector(
session_store.data_dir / "recorded-media-preparations"
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableLabV1SetupProjector | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
try:
portable_definition_registry = PortableRunDefinitionRegistry.from_file(
REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
)
portable_lab_v1_definition = next(
definition
for definition in portable_definition_registry.definitions
if definition.setup_id == PORTABLE_LAB_V1_SETUP_ID
)
portable_source_capability_service = RecordedK1SourceAdmissionService(
data_dir=session_store.data_dir,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
requirements=portable_lab_v1_definition.to_source_admission_requirements(),
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = PortableLabV1SetupProjector(
registry=portable_definition_registry,
capability_probe=portable_source_capability_service,
)
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = None
except (
PortableRunDefinitionRegistryError,
PortableSetupProjectionError,
OSError,
StopIteration,
ValueError,
) as exc:
# Portable LAB V1 is an optional observation-only slice. A drifted
# registry cannot affect K1, Simulation, legacy LAB, or the exact M49 queue.
OBSERVATORY_PORTABLE_SETUP_PROJECTOR = None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR = str(exc)
_ffmpeg = _resolve_media_tool("ffmpeg")
_ffprobe = _resolve_media_tool("ffprobe")
session_recorded_camera_frame_service = (
@@ -765,8 +841,23 @@ app.include_router(
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
portable_setup_projector=OBSERVATORY_PORTABLE_SETUP_PROJECTOR,
portable_setup_projector_error=OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR,
)
)
if (
OBSERVATORY_WORKER_PRODUCTION_API_ENABLED
and OBSERVATORY_WORKER_CLAIM_LEASE_READY
and OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
and OBSERVATORY_WORKER_AUTHENTICATION is not None
):
app.include_router(
build_observatory_worker_router(
OBSERVATORY_RECORDED_JOB_QUEUE,
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
)
)
app.include_router(
build_environment_router(root_provider=lambda: session_store.data_dir / "ui-environment")
)
@@ -1066,10 +1157,7 @@ app.include_router(
spatial_evidence_provider=m48_raw_evidence_reader,
evaluation_runner=LABORATORY_RUNNER,
evaluation_receipt_root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "laboratory-run-receipts"
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "laboratory-run-receipts"
),
)
)
@@ -1093,33 +1181,21 @@ app.include_router(
app.include_router(
build_m49_tgs_fail_closed_router(
root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "m49"
/ "tgs-fail-closed-results"
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-fail-closed-results"
),
)
)
app.include_router(
build_m49_tgs_full_shadow_router(
root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "m49"
/ "tgs-full-shadow-results"
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m49" / "tgs-full-shadow-results"
),
)
)
app.include_router(
build_vegetation_shadow_lab_router(
root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "lab-v1-vegetation"
/ "results"
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "lab-v1-vegetation" / "results"
),
canonical_recording_provider=_canonical_lab_recording_source,
camera_frame_provider=(
@@ -1128,9 +1204,7 @@ app.include_router(
else None
),
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
rerun_overlay_cache_root=(
session_store.data_dir / "laboratory-rerun-overlays"
),
rerun_overlay_cache_root=(session_store.data_dir / "laboratory-rerun-overlays"),
ffmpeg_path=_ffmpeg,
)
)
+60 -46
View File
@@ -22,6 +22,10 @@ from k1link.observatory.m49_queue_binding import (
M49QueueBindingIntegrityError,
M49RecordedQueueBindingService,
)
from k1link.observatory.portable_setup_projection import (
PortableLabV1SetupProjector,
PortableSetupProjectionError,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueCapacityError,
@@ -32,24 +36,24 @@ from k1link.observatory.recorded_jobs import (
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
from k1link.sessions.models import SessionSummary
OBSERVATORY_PROJECTION_SCHEMA: Literal[
OBSERVATORY_PROJECTION_SCHEMA: Literal["missioncore.observatory-lab-projection/v1"] = (
"missioncore.observatory-lab-projection/v1"
] = "missioncore.observatory-lab-projection/v1"
OBSERVATORY_RENAME_SCHEMA: Literal[
)
OBSERVATORY_RENAME_SCHEMA: Literal["missioncore.observatory-lab-projection-rename/v1"] = (
"missioncore.observatory-lab-projection-rename/v1"
] = "missioncore.observatory-lab-projection-rename/v1"
)
OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
"missioncore.observatory-run-preflight-request/v1"
] = "missioncore.observatory-run-preflight-request/v1"
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal["missioncore.observatory-run-preflight/v1"] = (
"missioncore.observatory-run-preflight/v1"
] = "missioncore.observatory-run-preflight/v1"
)
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
"missioncore.observatory-recorded-run-submit/v1"
] = "missioncore.observatory-recorded-run-submit/v1"
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal["missioncore.observatory-recorded-job-list/v1"] = (
"missioncore.observatory-recorded-job-list/v1"
] = "missioncore.observatory-recorded-job-list/v1"
)
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
"commands_enabled": False,
@@ -64,9 +68,7 @@ class _StrictApiModel(BaseModel):
class ObservatoryProjectionRenameRequest(_StrictApiModel):
schema_version: Literal[
"missioncore.observatory-lab-projection-rename/v1"
]
schema_version: Literal["missioncore.observatory-lab-projection-rename/v1"]
display_name: str = Field(min_length=1, max_length=160)
@@ -92,9 +94,7 @@ class ObservatoryRunPreflightRequest(_StrictApiModel):
class ObservatoryRunPreparationRequest(_StrictApiModel):
schema_version: Literal[
"missioncore.observatory-run-preparation-request/v1"
]
schema_version: Literal["missioncore.observatory-run-preparation-request/v1"]
idempotency_key: str = Field(
min_length=1,
max_length=160,
@@ -142,6 +142,8 @@ def build_observatory_router(
recorded_binding_service: M49RecordedQueueBindingService | None = None,
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
recorded_job_queue_error: str | None = None,
portable_setup_projector: PortableLabV1SetupProjector | None = None,
portable_setup_projector_error: str | None = None,
) -> APIRouter:
"""Build bounded catalog-only mutations for typed Observatory projections."""
@@ -212,6 +214,41 @@ def build_observatory_router(
available.add(result_id)
return frozenset(available)
if portable_setup_projector is not None:
@router.get("/api/v1/observatory/portable-laboratory-setups")
def list_observatory_portable_laboratory_setups(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
) -> dict[str, object]:
source = source_summary(source_session_id)
try:
return portable_setup_projector.catalog(source)
except PortableSetupProjectionError as exc:
raise HTTPException(
status_code=503,
detail="Portable-каталог LAB V1 нарушил контракт целостности.",
) from exc
elif portable_setup_projector_error is not None:
@router.get("/api/v1/observatory/portable-laboratory-setups")
def unavailable_observatory_portable_laboratory_setups(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
) -> None:
del source_session_id
raise HTTPException(
status_code=503,
detail="Portable-каталог LAB V1 недоступен.",
)
if setup_registry is not None:
@router.get("/api/v1/observatory/laboratory-setups")
@@ -225,9 +262,7 @@ def build_observatory_router(
source = source_summary(source_session_id)
return setup_registry.catalog(
source,
available_observatory_result_ids=available_observatory_results(
source_session_id
),
available_observatory_result_ids=available_observatory_results(source_session_id),
)
@router.post("/api/v1/observatory/run-preflights")
@@ -254,9 +289,7 @@ def build_observatory_router(
)
definition = projected["run_definition"]
expected_digest = (
definition.get("definition_sha256")
if isinstance(definition, dict)
else None
definition.get("definition_sha256") if isinstance(definition, dict) else None
)
if request.definition_sha256 != expected_digest:
raise HTTPException(
@@ -276,8 +309,7 @@ def build_observatory_router(
binding_service = recorded_binding_service
exact_queue_setup = (
binding_service is not None
and request.setup_id
== binding_service.config.setup.setup_id
and request.setup_id == binding_service.config.setup.setup_id
)
if (
compatible
@@ -303,9 +335,7 @@ def build_observatory_router(
{
"check_id": "source-compatibility",
"outcome": "pass" if compatible else "fail",
"reason_code": (
"source-compatible" if compatible else "source-incompatible"
),
"reason_code": ("source-compatible" if compatible else "source-incompatible"),
"message": (
"Источник точно совместим с сохранённым сетапом."
if compatible
@@ -323,11 +353,7 @@ def build_observatory_router(
{
"check_id": "executor",
"outcome": (
"not-applicable"
if existing
else "pass"
if queue_binding_ready
else "fail"
"not-applicable" if existing else "pass" if queue_binding_ready else "fail"
),
"reason_code": (
"existing-result-does-not-require-executor"
@@ -349,13 +375,7 @@ def build_observatory_router(
},
{
"check_id": "durable-queue",
"outcome": (
"not-applicable"
if existing
else "pass"
if queueable
else "fail"
),
"outcome": ("not-applicable" if existing else "pass" if queueable else "fail"),
"reason_code": (
"existing-result-does-not-require-queue"
if existing
@@ -378,9 +398,7 @@ def build_observatory_router(
"source_session_id": request.source_session_id,
"setup_id": request.setup_id,
"definition_sha256": expected_digest,
"outcome": (
"existing" if existing else "queueable" if queueable else "blocked"
),
"outcome": ("existing" if existing else "queueable" if queueable else "blocked"),
"submission_allowed": queueable,
"checks": checks,
"existing_result_ids": preflight.get("existing_result_ids", []),
@@ -421,9 +439,7 @@ def build_observatory_router(
request: ObservatoryRunPreparationRequest,
) -> dict[str, object]:
try:
existing = run_preparation_ledger.get_by_idempotency_key(
request.idempotency_key
)
existing = run_preparation_ledger.get_by_idempotency_key(request.idempotency_key)
request_sha256 = observatory_run_preparation_request_sha256(
idempotency_key=request.idempotency_key,
source_session_id=request.source_session_id,
@@ -634,9 +650,7 @@ def build_observatory_router(
request: ObservatoryRecordedRunSubmitRequest,
) -> dict[str, object]:
try:
existing_job = recorded_job_queue.get_by_idempotency_key(
request.idempotency_key
)
existing_job = recorded_job_queue.get_by_idempotency_key(request.idempotency_key)
except ObservatoryRecordedQueueNotFoundError:
existing_job = None
except (ObservatoryRecordedQueueError, ValueError) as exc:
+343
View File
@@ -0,0 +1,343 @@
"""Authenticated pull transport for the durable Observatory recorded-job queue.
The transport is deliberately narrower than an execution API. Worker callers
can claim a server-sealed RunDefinition and advance its durable state, but they
cannot supply commands, paths, environment variables, container images, or
priority. Those execution identities remain part of the queue-owned job.
"""
from __future__ import annotations
import hashlib
import hmac
import os
import re
import stat
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Annotated, Final, Literal
from fastapi import APIRouter, Depends, Header, HTTPException, Response
from fastapi import Path as ApiPath
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from pydantic import BaseModel, ConfigDict, Field
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedCheckpointError,
ObservatoryRecordedJobQueue,
ObservatoryRecordedPreemptionError,
ObservatoryRecordedQueueBusyError,
ObservatoryRecordedQueueCapacityError,
ObservatoryRecordedQueueConflictError,
ObservatoryRecordedQueueError,
ObservatoryRecordedQueueIntegrityError,
ObservatoryRecordedQueueNotFoundError,
ObservatoryRecordedQueueStaleClaimError,
)
OBSERVATORY_WORKER_CLAIM_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-claim-request/v1"
OBSERVATORY_WORKER_START_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-start-request/v1"
OBSERVATORY_WORKER_CHECKPOINT_REQUEST_SCHEMA: Final = (
"missioncore.observatory-worker-checkpoint-request/v1"
)
OBSERVATORY_WORKER_SUCCEED_REQUEST_SCHEMA: Final = (
"missioncore.observatory-worker-succeed-request/v1"
)
OBSERVATORY_WORKER_FAIL_REQUEST_SCHEMA: Final = "missioncore.observatory-worker-fail-request/v1"
OBSERVATORY_WORKER_CONTOUR_HEADER: Final = "X-Mission-Core-Contour-Id"
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_JOB_ID_PATTERN = r"^observatory-run-[a-f0-9]{32}$"
_CLAIM_TOKEN_PATTERN = r"^[a-f0-9]{64}$"
_CLAIM_REQUEST_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$"
_SESSION_ID_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
_IDENTIFIER_PATTERN = r"^[a-z][a-z0-9-]{2,95}$"
_WORKER_BEARER = HTTPBearer(auto_error=False)
_TOKEN = re.compile(r"^[A-Za-z0-9._:-]{32,512}$")
@dataclass(frozen=True, slots=True)
class ObservatoryWorkerAuthentication:
"""Server-owned Worker identity and the SHA-256 of its bearer secret."""
bearer_token_sha256: str
contour_id: str = "worker-006"
def __post_init__(self) -> None:
if _SHA256.fullmatch(self.bearer_token_sha256) is None:
raise ValueError("Worker bearer token SHA-256 is invalid")
if _IDENTIFIER.fullmatch(self.contour_id) is None:
raise ValueError("Worker contour id is invalid")
def load_observatory_worker_authentication(
token_path: Path,
*,
contour_id: str = "worker-006",
) -> ObservatoryWorkerAuthentication:
"""Load one local Worker credential without retaining its plaintext.
The credential file is an operator/deployment concern. Mission Core only
retains its SHA-256 in the router configuration and refuses symlinks,
non-regular files, or group/other permissions.
"""
candidate = token_path.expanduser().absolute()
descriptor: int | None = None
try:
descriptor = os.open(
candidate,
os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0),
)
metadata = os.fstat(descriptor)
if not stat.S_ISREG(metadata.st_mode):
raise ValueError("Worker bearer credential must be a regular file")
if metadata.st_mode & 0o077:
raise ValueError("Worker bearer credential permissions are too broad")
if not 32 <= metadata.st_size <= 512:
raise ValueError("Worker bearer credential format is invalid")
with os.fdopen(descriptor, "rb") as stream:
descriptor = None
payload = stream.read(513)
except ValueError:
raise
except OSError as exc:
raise ValueError("Worker bearer credential is unavailable") from exc
finally:
if descriptor is not None:
os.close(descriptor)
try:
token = payload.decode("ascii")
except UnicodeDecodeError as exc:
raise ValueError("Worker bearer credential is not ASCII") from exc
if _TOKEN.fullmatch(token) is None:
raise ValueError("Worker bearer credential format is invalid")
return ObservatoryWorkerAuthentication(
bearer_token_sha256=hashlib.sha256(payload).hexdigest(),
contour_id=contour_id,
)
class _StrictWorkerRequest(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class ObservatoryWorkerClaimRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-claim-request/v1"]
claim_request_id: str = Field(
min_length=1,
max_length=160,
pattern=_CLAIM_REQUEST_ID_PATTERN,
)
class ObservatoryWorkerStartRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-start-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
class ObservatoryWorkerCheckpointRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-checkpoint-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
checkpoint_id: str = Field(
min_length=3,
max_length=96,
pattern=_IDENTIFIER_PATTERN,
)
class ObservatoryWorkerSucceedRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-succeed-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
result_id: str = Field(
min_length=1,
max_length=128,
pattern=_SESSION_ID_PATTERN,
)
result_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
class ObservatoryWorkerFailRequest(_StrictWorkerRequest):
schema_version: Literal["missioncore.observatory-worker-fail-request/v1"]
claim_token: str = Field(pattern=_CLAIM_TOKEN_PATTERN)
error_code: str = Field(
min_length=3,
max_length=96,
pattern=_IDENTIFIER_PATTERN,
)
message: str = Field(min_length=1, max_length=1_000)
def build_observatory_worker_router(
queue: ObservatoryRecordedJobQueue,
*,
authentication: ObservatoryWorkerAuthentication,
) -> APIRouter:
"""Build the bounded Worker pull/state-transition router.
``authentication`` contains only a token digest. The plaintext bearer
secret exists transiently while FastAPI parses one request, is immediately
hashed, and is compared to the configured digest in constant time.
"""
def require_configured_worker(
credentials: Annotated[
HTTPAuthorizationCredentials | None,
Depends(_WORKER_BEARER),
],
contour_id: Annotated[
str | None,
Header(alias=OBSERVATORY_WORKER_CONTOUR_HEADER),
] = None,
) -> None:
if credentials is None or credentials.scheme.lower() != "bearer":
raise _unauthorized()
token = credentials.credentials
if not token or len(token) > 512:
raise _unauthorized()
supplied_sha256 = hashlib.sha256(token.encode("utf-8")).hexdigest()
if not hmac.compare_digest(
supplied_sha256,
authentication.bearer_token_sha256,
):
raise _unauthorized()
if contour_id is None or not hmac.compare_digest(
contour_id,
authentication.contour_id,
):
raise HTTPException(
status_code=403,
detail="Worker contour identity was rejected.",
)
router = APIRouter(
prefix="/api/v1/worker/observatory",
tags=["observatory-worker"],
dependencies=[Depends(require_configured_worker)],
)
@router.post("/recorded-jobs/claims", response_model=None)
def claim_next(
request: ObservatoryWorkerClaimRequest,
) -> dict[str, object] | Response:
claim = _queue_call(
lambda: queue.claim_next(
claimant_id=authentication.contour_id,
claim_request_id=request.claim_request_id,
)
)
if claim is None:
return Response(status_code=204)
return claim.as_dict()
@router.get("/recorded-jobs/{job_id}")
def get_job(
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(lambda: queue.get(job_id)).as_dict()
@router.post("/recorded-jobs/{job_id}/start")
def start_job(
request: ObservatoryWorkerStartRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(lambda: queue.start(job_id, claim_token=request.claim_token)).as_dict()
@router.post("/recorded-jobs/{job_id}/checkpoint")
def checkpoint_job(
request: ObservatoryWorkerCheckpointRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(
lambda: queue.checkpoint(
job_id,
claim_token=request.claim_token,
checkpoint_id=request.checkpoint_id,
)
).as_dict()
@router.post("/recorded-jobs/{job_id}/succeed")
def succeed_job(
request: ObservatoryWorkerSucceedRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(
lambda: queue.succeed(
job_id,
claim_token=request.claim_token,
result_id=request.result_id,
result_sha256=request.result_sha256,
)
).as_dict()
@router.post("/recorded-jobs/{job_id}/fail")
def fail_job(
request: ObservatoryWorkerFailRequest,
job_id: Annotated[str, ApiPath(pattern=_JOB_ID_PATTERN)],
) -> dict[str, object]:
return _queue_call(
lambda: queue.fail(
job_id,
claim_token=request.claim_token,
error_code=request.error_code,
message=request.message,
)
).as_dict()
return router
def _unauthorized() -> HTTPException:
return HTTPException(
status_code=401,
detail="Worker bearer credential was rejected.",
headers={"WWW-Authenticate": "Bearer"},
)
def _queue_call[T](operation: Callable[[], T]) -> T:
try:
return operation()
except ObservatoryRecordedQueueNotFoundError as exc:
raise HTTPException(
status_code=404,
detail="Recorded job was not found.",
) from exc
except ObservatoryRecordedQueueStaleClaimError as exc:
raise HTTPException(
status_code=409,
detail="Recorded-job claim is stale.",
) from exc
except ObservatoryRecordedCheckpointError as exc:
raise HTTPException(
status_code=409,
detail="Recorded-job checkpoint was rejected.",
) from exc
except ObservatoryRecordedQueueConflictError as exc:
raise HTTPException(
status_code=409,
detail="Recorded-job transition conflicts with durable state.",
) from exc
except (ObservatoryRecordedQueueBusyError, ObservatoryRecordedPreemptionError) as exc:
raise HTTPException(
status_code=409,
detail="Recorded-job resources are reserved for live K1 work.",
) from exc
except ObservatoryRecordedQueueCapacityError as exc:
raise HTTPException(
status_code=503,
detail="Recorded-job queue capacity is unavailable.",
headers={"Retry-After": "5"},
) from exc
except ObservatoryRecordedQueueIntegrityError as exc:
raise HTTPException(
status_code=503,
detail="Recorded-job queue integrity is unavailable.",
) from exc
except ObservatoryRecordedQueueError as exc:
raise HTTPException(
status_code=503,
detail="Recorded-job queue is unavailable.",
) from exc