SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(observatory): add durable recorded compute queue

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 01:04:25 +03:00
parent e72609120f
commit d67e86176e
23 changed files with 6300 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/observatory/m49_queue_binding.py
+1201
View File
File diff suppressed because it is too large Load Diff
src/k1link/observatory/recorded_jobs.py
+2164
View File
File diff suppressed because it is too large Load Diff
src/k1link/web/app.py
+40
View File
@@ -42,6 +42,15 @@ from k1link.observatory import (
ObservatoryRunPreparationLedger,
load_observatory_run_preparation_ledger,
)
from k1link.observatory.m49_queue_binding import (
M49QueueBindingConfig,
M49QueueBindingError,
M49RecordedQueueBindingService,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueError,
)
from k1link.sessions import (
MaterializedRecording,
RecordedCameraFrameService,
@@ -242,6 +251,34 @@ OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR: str | None
OBSERVATORY_RUN_PREPARATION_LEDGER,
OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
) = load_observatory_run_preparation_ledger(session_store.data_dir)
OBSERVATORY_RECORDED_BINDING_SERVICE: M49RecordedQueueBindingService | None
OBSERVATORY_RECORDED_JOB_QUEUE: ObservatoryRecordedJobQueue | None
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR: str | None
try:
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is None:
raise M49QueueBindingError("observatory setup registry is unavailable")
OBSERVATORY_RECORDED_BINDING_SERVICE = M49RecordedQueueBindingService(
data_dir=session_store.data_dir,
session_store=session_store,
setup_registry=OBSERVATORY_LABORATORY_SETUP_REGISTRY,
config=M49QueueBindingConfig.from_file(
REPOSITORY_ROOT
/ "config"
/ "observatory-m49-recorded-queue-binding.json"
),
)
OBSERVATORY_RECORDED_JOB_QUEUE = ObservatoryRecordedJobQueue(
session_store.data_dir,
definitions=OBSERVATORY_RECORDED_BINDING_SERVICE.definitions,
)
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = None
except (M49QueueBindingError, ObservatoryRecordedQueueError, OSError, ValueError) as exc:
# Recorded execution remains an optional observation-only slice. A drifted
# seal or queue must fail closed without preventing K1, Simulation or legacy
# LAB from starting.
OBSERVATORY_RECORDED_BINDING_SERVICE = None
OBSERVATORY_RECORDED_JOB_QUEUE = None
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR = str(exc)
simulation_project_store = SimulationProjectStore(session_store.data_dir)
simulation_project_service = SimulationProjectService(simulation_project_store)
session_artifact_gateway = configured_artifact_gateway(session_store.data_dir)
@@ -725,6 +762,9 @@ app.include_router(
setup_registry_error=OBSERVATORY_LABORATORY_SETUP_REGISTRY_ERROR,
run_preparation_ledger=OBSERVATORY_RUN_PREPARATION_LEDGER,
run_preparation_ledger_error=OBSERVATORY_RUN_PREPARATION_LEDGER_ERROR,
recorded_binding_service=OBSERVATORY_RECORDED_BINDING_SERVICE,
recorded_job_queue=OBSERVATORY_RECORDED_JOB_QUEUE,
recorded_job_queue_error=OBSERVATORY_RECORDED_JOB_QUEUE_ERROR,
)
)
app.include_router(
src/k1link/web/observatory_api.py
+348 -3
View File
@@ -17,6 +17,18 @@ from k1link.observatory import (
is_admitted_observatory_recorded_result,
observatory_run_preparation_request_sha256,
)
from k1link.observatory.m49_queue_binding import (
M49QueueBindingError,
M49QueueBindingIntegrityError,
M49RecordedQueueBindingService,
)
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueCapacityError,
ObservatoryRecordedQueueConflictError,
ObservatoryRecordedQueueError,
ObservatoryRecordedQueueNotFoundError,
)
from k1link.sessions import SessionIntegrityError, SessionNotFoundError, SessionStore
from k1link.sessions.models import SessionSummary
@@ -32,6 +44,19 @@ OBSERVATORY_RUN_PREFLIGHT_REQUEST_SCHEMA: Literal[
OBSERVATORY_RUN_PREFLIGHT_SCHEMA: Literal[
"missioncore.observatory-run-preflight/v1"
] = "missioncore.observatory-run-preflight/v1"
OBSERVATORY_RECORDED_RUN_SUBMIT_SCHEMA: Literal[
"missioncore.observatory-recorded-run-submit/v1"
] = "missioncore.observatory-recorded-run-submit/v1"
OBSERVATORY_RECORDED_JOB_LIST_SCHEMA: Literal[
"missioncore.observatory-recorded-job-list/v1"
] = "missioncore.observatory-recorded-job-list/v1"
_OBSERVATION_ONLY_AUTHORITY: dict[str, bool] = {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
}
class _StrictApiModel(BaseModel):
@@ -88,6 +113,25 @@ class ObservatoryRunPreparationRequest(_StrictApiModel):
definition_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
class ObservatoryRecordedRunSubmitRequest(_StrictApiModel):
schema_version: Literal["missioncore.observatory-recorded-run-submit/v1"]
idempotency_key: str = Field(
min_length=1,
max_length=160,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$",
)
source_session_id: str = Field(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
)
setup_id: str = Field(
min_length=3,
max_length=96,
pattern=r"^[a-z][a-z0-9-]{2,95}$",
)
def build_observatory_router(
store: SessionStore,
*,
@@ -95,6 +139,9 @@ def build_observatory_router(
setup_registry_error: str | None = None,
run_preparation_ledger: ObservatoryRunPreparationLedger | None = None,
run_preparation_ledger_error: str | None = None,
recorded_binding_service: M49RecordedQueueBindingService | None = None,
recorded_job_queue: ObservatoryRecordedJobQueue | None = None,
recorded_job_queue_error: str | None = None,
) -> APIRouter:
"""Build bounded catalog-only mutations for typed Observatory projections."""
@@ -224,6 +271,34 @@ def build_observatory_router(
assert isinstance(executor, dict)
compatible = compatibility.get("compatible") is True
existing = preflight.get("outcome") == "existing"
queue_binding_ready = False
queue_binding_reason: str | None = None
binding_service = recorded_binding_service
exact_queue_setup = (
binding_service is not None
and request.setup_id
== binding_service.config.setup.setup_id
)
if (
compatible
and not existing
and binding_service is not None
and exact_queue_setup
and isinstance(expected_digest, str)
):
try:
binding_service.check(
source_session_id=request.source_session_id,
setup_id=request.setup_id,
definition_sha256=expected_digest,
)
queue_binding_ready = True
except M49QueueBindingError:
queue_binding_reason = (
"Точная привязка источника и исполняемого релиза не прошла "
"проверку целостности."
)
queueable = queue_binding_ready and recorded_job_queue is not None
checks: list[dict[str, Any]] = [
{
"check_id": "source-compatibility",
@@ -247,26 +322,66 @@ def build_observatory_router(
},
{
"check_id": "executor",
"outcome": "not-applicable" if existing else "fail",
"outcome": (
"not-applicable"
if existing
else "pass"
if queue_binding_ready
else "fail"
),
"reason_code": (
"existing-result-does-not-require-executor"
if existing
else "executor-release-sealed"
if queue_binding_ready
else str(executor.get("reason_code"))
),
"message": (
"Готовый результат открывается без повторного запуска Worker."
if existing
else (
"Исполняемый релиз и его ресурсы запечатаны; отдельный "
"Worker service заберёт расчёт из очереди после установки."
)
if queue_binding_ready
else str(executor.get("reason"))
),
},
{
"check_id": "durable-queue",
"outcome": (
"not-applicable"
if existing
else "pass"
if queueable
else "fail"
),
"reason_code": (
"existing-result-does-not-require-queue"
if existing
else "durable-queue-ready"
if queueable
else "durable-queue-unavailable"
),
"message": (
"Готовый результат не требует постановки в очередь."
if existing
else "Durable-очередь готова принять расчёт."
if queueable
else queue_binding_reason
or "Этот источник и сетап пока нельзя поставить в очередь."
),
},
]
return {
"schema_version": OBSERVATORY_RUN_PREFLIGHT_SCHEMA,
"source_session_id": request.source_session_id,
"setup_id": request.setup_id,
"definition_sha256": expected_digest,
"outcome": "existing" if existing else "blocked",
"submission_allowed": False,
"outcome": (
"existing" if existing else "queueable" if queueable else "blocked"
),
"submission_allowed": queueable,
"checks": checks,
"existing_result_ids": preflight.get("existing_result_ids", []),
"executor": executor,
@@ -508,6 +623,236 @@ def build_observatory_router(
detail="Подготовка расчётов Обсерватории недоступна.",
)
if (
setup_registry is not None
and recorded_binding_service is not None
and recorded_job_queue is not None
):
@router.post("/api/v1/observatory/runs", status_code=202)
def submit_observatory_recorded_run(
request: ObservatoryRecordedRunSubmitRequest,
) -> dict[str, object]:
try:
existing_job = recorded_job_queue.get_by_idempotency_key(
request.idempotency_key
)
except ObservatoryRecordedQueueNotFoundError:
existing_job = None
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
) from exc
if existing_job is not None:
if (
existing_job.source_session_id != request.source_session_id
or existing_job.setup_id != request.setup_id
):
raise HTTPException(
status_code=409,
detail="Ключ идемпотентности уже связан с другим расчётом.",
)
return existing_job.as_dict()
source = source_summary(request.source_session_id)
try:
setup_registry.setup(request.setup_id)
except KeyError as exc:
raise HTTPException(
status_code=404,
detail="Сетап лаборатории не найден.",
) from exc
catalog = setup_registry.catalog(
source,
available_observatory_result_ids=available_observatory_results(
request.source_session_id
),
)
setups = catalog.get("setups")
if not isinstance(setups, list):
raise HTTPException(
status_code=503,
detail="Каталог сетапов Обсерватории нарушил контракт.",
)
projected = next(
(
item
for item in setups
if isinstance(item, dict) and item.get("setup_id") == request.setup_id
),
None,
)
if projected is None:
raise HTTPException(
status_code=503,
detail="Каталог сетапов Обсерватории нарушил контракт.",
)
compatibility = projected.get("compatibility")
preflight = projected.get("preflight")
definition = projected.get("run_definition")
if not isinstance(compatibility, dict) or not isinstance(preflight, dict):
raise HTTPException(
status_code=503,
detail="Каталог сетапов Обсерватории нарушил контракт.",
)
if compatibility.get("compatible") is not True:
raise HTTPException(
status_code=409,
detail="Исходная сессия несовместима с выбранным сетапом.",
)
if preflight.get("outcome") == "existing":
raise HTTPException(
status_code=409,
detail="Точный результат уже существует; новый расчёт не создан.",
)
if not isinstance(definition, dict):
raise HTTPException(
status_code=409,
detail="Для сетапа нет воспроизводимой RunDefinition.",
)
definition_sha256 = definition.get("definition_sha256")
if not isinstance(definition_sha256, str):
raise HTTPException(
status_code=503,
detail="Каталог сетапов Обсерватории нарушил контракт.",
)
try:
admission = recorded_binding_service.admit(
source_session_id=request.source_session_id,
setup_id=request.setup_id,
definition_sha256=definition_sha256,
)
job, _created = recorded_job_queue.submit(
admission.intent(idempotency_key=request.idempotency_key),
enqueue=True,
)
return job.as_dict()
except M49QueueBindingIntegrityError as exc:
raise HTTPException(
status_code=409,
detail=(
"Точная привязка источника и исполняемого релиза не прошла "
"проверку целостности."
),
) from exc
except ObservatoryRecordedQueueConflictError as exc:
try:
existing_job = recorded_job_queue.get_by_idempotency_key(
request.idempotency_key
)
except ObservatoryRecordedQueueNotFoundError:
existing_job = None
except (ObservatoryRecordedQueueError, ValueError) as lookup_exc:
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
) from lookup_exc
if (
existing_job is not None
and existing_job.source_session_id == request.source_session_id
and existing_job.setup_id == request.setup_id
):
return existing_job.as_dict()
raise HTTPException(
status_code=409,
detail="Ключ идемпотентности уже связан с другим расчётом.",
) from exc
except ObservatoryRecordedQueueCapacityError as exc:
raise HTTPException(
status_code=503,
detail="Квота durable-очереди расчётов исчерпана.",
) from exc
except (M49QueueBindingError, ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
) from exc
@router.get("/api/v1/observatory/runs")
def list_observatory_recorded_runs(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
setup_id: str = Query(
min_length=3,
max_length=96,
pattern=r"^[a-z][a-z0-9-]{2,95}$",
),
limit: int = Query(default=20, ge=1, le=100),
) -> dict[str, object]:
try:
jobs = recorded_job_queue.list_jobs(
source_session_id=source_session_id,
setup_id=setup_id,
limit=limit,
)
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
) from exc
return {
"schema_version": OBSERVATORY_RECORDED_JOB_LIST_SCHEMA,
"items": [job.as_dict() for job in jobs],
"authority": dict(_OBSERVATION_ONLY_AUTHORITY),
}
@router.get("/api/v1/observatory/runs/{job_id}")
def get_observatory_recorded_run(
job_id: str = ApiPath(
min_length=48,
max_length=48,
pattern=r"^observatory-run-[a-f0-9]{32}$",
),
) -> dict[str, object]:
try:
return recorded_job_queue.get(job_id).as_dict()
except ObservatoryRecordedQueueNotFoundError as exc:
raise HTTPException(
status_code=404,
detail="Расчёт Обсерватории не найден.",
) from exc
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
) from exc
elif recorded_job_queue_error is not None:
@router.post("/api/v1/observatory/runs")
def unavailable_observatory_recorded_run(
request: ObservatoryRecordedRunSubmitRequest,
) -> None:
del request
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
)
@router.get("/api/v1/observatory/runs")
def unavailable_observatory_recorded_runs(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
setup_id: str = Query(
min_length=3,
max_length=96,
pattern=r"^[a-z][a-z0-9-]{2,95}$",
),
limit: int = Query(default=20, ge=1, le=100),
) -> None:
del source_session_id, setup_id, limit
raise HTTPException(
status_code=503,
detail="Durable-очередь расчётов недоступна.",
)
@router.patch(
"/api/v1/observatory/lab-projections/{session_id}",
response_model=ObservatoryProjectionDocument,