feat(observatory): add durable recorded compute queue

apps/control-station/src/core/observatory/laboratorySetups.ts

@@ -61,8 +61,8 @@ export interface ObservatoryLaboratoryRunPreflight {
   readonly sourceSessionId: string;
   readonly setupId: string;
   readonly definitionSha256: string | null;
-  readonly outcome: "existing" | "blocked";
-  readonly submissionAllowed: false;
+  readonly outcome: "existing" | "queueable" | "blocked";
+  readonly submissionAllowed: boolean;
   readonly checks: readonly {
     readonly checkId: string;
     readonly outcome: "pass" | "fail" | "not-applicable";
@@ -287,8 +287,15 @@ function decodePreflight(value: unknown): ObservatoryLaboratoryRunPreflight {
     sourceSessionId: text(row.source_session_id, "preflight source_session_id"),
     setupId: text(row.setup_id, "preflight setup_id"),
     definitionSha256: digest,
-    outcome: oneOf(row.outcome, ["existing", "blocked"] as const, "preflight outcome"),
-    submissionAllowed: exact(row.submission_allowed, false, "preflight submission_allowed"),
+    outcome: oneOf(
+      row.outcome,
+      ["existing", "queueable", "blocked"] as const,
+      "preflight outcome",
+    ),
+    submissionAllowed: boolean(
+      row.submission_allowed,
+      "preflight submission_allowed",
+    ),
     checks: array(row.checks, "preflight checks").map((item) => {
       const check = record(item, "preflight check");
       exactKeys(check, ["check_id", "message", "outcome", "reason_code"], "preflight check");

apps/control-station/src/core/observatory/recordedJobs.ts

@@ -0,0 +1,241 @@
+const SUBMIT_SCHEMA = "missioncore.observatory-recorded-run-submit/v1";
+const JOB_SCHEMA = "missioncore.observatory-recorded-job/v1";
+const JOB_LIST_SCHEMA = "missioncore.observatory-recorded-job-list/v1";
+const SHA256 = /^[a-f0-9]{64}$/;
+
+export type ObservatoryRecordedJobState =
+  | "accepted"
+  | "queued"
+  | "claimed"
+  | "running"
+  | "paused"
+  | "preemption-pending"
+  | "succeeded"
+  | "failed"
+  | "reconciliation-required";
+
+export interface ObservatoryRecordedJob {
+  readonly jobId: string;
+  readonly idempotencyKey: string;
+  readonly sourceSessionId: string;
+  readonly setupId: string;
+  readonly definitionSha256: string;
+  readonly state: ObservatoryRecordedJobState;
+  readonly restartFromZero: boolean;
+  readonly resultId: string | null;
+  readonly terminalMessage: string | null;
+  readonly createdAtUtc: string;
+  readonly updatedAtUtc: string;
+}
+
+export type ObservatoryRecordedJobFetch = (
+  input: RequestInfo | URL,
+  init?: RequestInit,
+) => Promise<Response>;
+
+export class ObservatoryRecordedJobContractError extends Error {
+  readonly status: number | null;
+
+  constructor(message: string, status: number | null = null) {
+    super(message);
+    this.name = "ObservatoryRecordedJobContractError";
+    this.status = status;
+  }
+}
+
+export async function fetchObservatoryRecordedJobs(
+  sourceSessionId: string,
+  setupId: string,
+  {
+    signal,
+    fetcher = globalThis.fetch,
+  }: {
+    signal?: AbortSignal;
+    fetcher?: ObservatoryRecordedJobFetch;
+  } = {},
+): Promise<readonly ObservatoryRecordedJob[]> {
+  const query = new URLSearchParams({
+    source_session_id: sourceSessionId,
+    setup_id: setupId,
+    limit: "20",
+  });
+  const response = await request(fetcher, `/api/v1/observatory/runs?${query}`, {
+    method: "GET",
+    headers: { Accept: "application/json" },
+    signal,
+  });
+  const body = await responseBody(response);
+  if (!response.ok) throw apiError(body, response.status);
+  const row = record(body, "список расчётов");
+  exactKeys(row, ["authority", "items", "schema_version"], "список расчётов");
+  exact(row.schema_version, JOB_LIST_SCHEMA, "schema_version списка расчётов");
+  observationAuthority(row.authority);
+  return array(row.items, "items").map(decodeJob).filter((job) => (
+    job.sourceSessionId === sourceSessionId && job.setupId === setupId
+  ));
+}
+
+export async function submitObservatoryRecordedJob(
+  sourceSessionId: string,
+  setupId: string,
+  idempotencyKey: string,
+  {
+    signal,
+    fetcher = globalThis.fetch,
+  }: {
+    signal?: AbortSignal;
+    fetcher?: ObservatoryRecordedJobFetch;
+  } = {},
+): Promise<ObservatoryRecordedJob> {
+  const response = await request(fetcher, "/api/v1/observatory/runs", {
+    method: "POST",
+    headers: { Accept: "application/json", "Content-Type": "application/json" },
+    body: JSON.stringify({
+      schema_version: SUBMIT_SCHEMA,
+      idempotency_key: idempotencyKey,
+      source_session_id: sourceSessionId,
+      setup_id: setupId,
+    }),
+    signal,
+  });
+  const body = await responseBody(response);
+  if (!response.ok) throw apiError(body, response.status);
+  const job = decodeJob(body);
+  if (job.sourceSessionId !== sourceSessionId || job.setupId !== setupId) {
+    throw new ObservatoryRecordedJobContractError(
+      "Расчёт относится к другой сессии или сетапу.",
+    );
+  }
+  return job;
+}
+
+function decodeJob(value: unknown): ObservatoryRecordedJob {
+  const row = record(value, "расчёт");
+  exactKeys(row, [
+    "authority", "checkpoint_policy", "claim_generation", "created_at_utc", "executor",
+    "idempotency_key", "identity_sha256", "job_id", "preemption_receipt_sha256",
+    "preemption_requested", "priority", "request_sha256", "restart_from_zero", "result",
+    "schema_version", "setup", "source", "state", "submission_receipt_sha256", "terminal",
+    "updated_at_utc",
+  ], "расчёт");
+  exact(row.schema_version, JOB_SCHEMA, "schema_version расчёта");
+  observationAuthority(row.authority);
+  digest(row.request_sha256, "request_sha256");
+  digest(row.identity_sha256, "identity_sha256");
+  digest(row.submission_receipt_sha256, "submission_receipt_sha256");
+  const source = record(row.source, "source");
+  exactKeys(source, [
+    "adapter", "bundle_sha256", "capability_manifest_sha256", "catalog_sha256", "session_id",
+  ], "source");
+  digest(source.catalog_sha256, "source.catalog_sha256");
+  digest(source.bundle_sha256, "source.bundle_sha256");
+  digest(source.capability_manifest_sha256, "source.capability_manifest_sha256");
+  const setup = record(row.setup, "setup");
+  exactKeys(setup, ["definition_id", "definition_sha256", "definition_version", "setup_id"], "setup");
+  digest(setup.definition_sha256, "setup.definition_sha256");
+  const state = oneOf(row.state, [
+    "accepted", "queued", "claimed", "running", "paused", "preemption-pending",
+    "succeeded", "failed", "reconciliation-required",
+  ] as const, "state");
+  const result = row.result === null ? null : record(row.result, "result");
+  if (result !== null) exactKeys(result, ["result_id", "sha256"], "result");
+  const terminal = row.terminal === null ? null : record(row.terminal, "terminal");
+  if (terminal !== null) exactKeys(terminal, ["code", "message"], "terminal");
+  return {
+    jobId: text(row.job_id, "job_id"),
+    idempotencyKey: text(row.idempotency_key, "idempotency_key"),
+    sourceSessionId: text(source.session_id, "source.session_id"),
+    setupId: text(setup.setup_id, "setup.setup_id"),
+    definitionSha256: String(setup.definition_sha256),
+    state,
+    restartFromZero: boolean(row.restart_from_zero, "restart_from_zero"),
+    resultId: result === null ? null : text(result.result_id, "result.result_id"),
+    terminalMessage: terminal === null || terminal.message === null
+      ? null
+      : text(terminal.message, "terminal.message"),
+    createdAtUtc: text(row.created_at_utc, "created_at_utc"),
+    updatedAtUtc: text(row.updated_at_utc, "updated_at_utc"),
+  };
+}
+
+function observationAuthority(value: unknown): void {
+  const row = record(value, "authority");
+  const keys = ["commands_enabled", "actuation_allowed", "navigation_or_safety_accepted", "production_accepted"];
+  exactKeys(row, keys, "authority");
+  keys.forEach((key) => exact(row[key], false, `authority.${key}`));
+}
+
+async function request(fetcher: ObservatoryRecordedJobFetch, input: string, init: RequestInit): Promise<Response> {
+  try {
+    return await fetcher(input, init);
+  } catch (error) {
+    if (error instanceof DOMException && error.name === "AbortError") throw error;
+    throw new ObservatoryRecordedJobContractError("Очередь расчётов недоступна.");
+  }
+}
+
+async function responseBody(response: Response): Promise<unknown> {
+  const body = await response.text();
+  if (!body) return undefined;
+  try { return JSON.parse(body) as unknown; } catch { return body; }
+}
+
+function apiError(body: unknown, status: number): ObservatoryRecordedJobContractError {
+  const detail = body && typeof body === "object" && !Array.isArray(body)
+    ? (body as Record<string, unknown>).detail
+    : null;
+  return new ObservatoryRecordedJobContractError(
+    typeof detail === "string" && detail.trim() ? detail : `Observatory API вернул HTTP ${status}.`,
+    status,
+  );
+}
+
+function record(value: unknown, label: string): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new ObservatoryRecordedJobContractError(`${label}: ожидался объект.`);
+  }
+  return value as Record<string, unknown>;
+}
+
+function array(value: unknown, label: string): unknown[] {
+  if (!Array.isArray(value)) throw new ObservatoryRecordedJobContractError(`${label}: ожидался массив.`);
+  return value;
+}
+
+function text(value: unknown, label: string): string {
+  if (typeof value !== "string" || !value.trim()) {
+    throw new ObservatoryRecordedJobContractError(`${label}: ожидался текст.`);
+  }
+  return value;
+}
+
+function digest(value: unknown, label: string): string {
+  const valueText = text(value, label);
+  if (!SHA256.test(valueText)) throw new ObservatoryRecordedJobContractError(`${label}: некорректный digest.`);
+  return valueText;
+}
+
+function boolean(value: unknown, label: string): boolean {
+  if (typeof value !== "boolean") throw new ObservatoryRecordedJobContractError(`${label}: ожидался boolean.`);
+  return value;
+}
+
+function exact<T>(value: unknown, expected: T, label: string): T {
+  if (value !== expected) throw new ObservatoryRecordedJobContractError(`${label}: значение изменилось.`);
+  return expected;
+}
+
+function oneOf<const T extends readonly string[]>(value: unknown, allowed: T, label: string): T[number] {
+  if (typeof value !== "string" || !allowed.includes(value)) {
+    throw new ObservatoryRecordedJobContractError(`${label}: значение не поддерживается.`);
+  }
+  return value as T[number];
+}
+
+function exactKeys(value: Record<string, unknown>, expected: readonly string[], label: string): void {
+  const actual = Object.keys(value).sort();
+  const sortedExpected = [...expected].sort();
+  if (actual.length !== sortedExpected.length || actual.some((key, index) => key !== sortedExpected[index])) {
+    throw new ObservatoryRecordedJobContractError(`${label}: обнаружены неизвестные поля.`);
+  }
+}

apps/control-station/src/core/observatory/useObservatoryLaboratorySetups.ts

@@ -111,6 +111,11 @@ export function useObservatoryLaboratorySetups(sourceSessionId: string) {
     }
   }, [preflight.kind, selectedSetup, sourceSessionId]);
 
+  useEffect(() => {
+    if (state !== "ready" || !selectedSetup || preflight.kind !== "idle") return;
+    void check();
+  }, [check, preflight.kind, selectedSetup, state]);
+
   return {
     catalog: activeCatalog,
     state,

apps/control-station/src/core/observatory/useObservatoryRecordedJobs.ts

@@ -0,0 +1,112 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import {
+  fetchObservatoryRecordedJobs,
+  submitObservatoryRecordedJob,
+  type ObservatoryRecordedJob,
+} from "./recordedJobs";
+
+type RecordedJobsState = "idle" | "loading" | "ready" | "refreshing" | "submitting" | "error";
+
+const OPEN_STATES = new Set([
+  "accepted", "queued", "claimed", "running", "paused", "preemption-pending",
+  "reconciliation-required",
+]);
+
+export function useObservatoryRecordedJobs(sourceSessionId: string, setupId: string) {
+  const [jobs, setJobs] = useState<readonly ObservatoryRecordedJob[]>([]);
+  const [state, setState] = useState<RecordedJobsState>("idle");
+  const [error, setError] = useState<string | null>(null);
+  const [revision, setRevision] = useState(0);
+  const requestRef = useRef<AbortController | null>(null);
+  const requestSequence = useRef(0);
+  const idempotencyKeys = useRef(new Map<string, string>());
+  const selectionKey = `${sourceSessionId}\u0000${setupId}`;
+
+  useEffect(() => {
+    requestRef.current?.abort();
+    requestRef.current = null;
+    if (!sourceSessionId || !setupId) {
+      setJobs([]);
+      setState("idle");
+      setError(null);
+      return;
+    }
+    const sequence = ++requestSequence.current;
+    const request = new AbortController();
+    requestRef.current = request;
+    setState((current) => jobs.length > 0 && current !== "idle" ? "refreshing" : "loading");
+    setError(null);
+    void fetchObservatoryRecordedJobs(sourceSessionId, setupId, { signal: request.signal })
+      .then((next) => {
+        if (request.signal.aborted || requestSequence.current !== sequence) return;
+        setJobs(next);
+        setState("ready");
+      })
+      .catch((caught: unknown) => {
+        if (request.signal.aborted || requestSequence.current !== sequence) return;
+        setState("error");
+        setError(caught instanceof Error && caught.message.trim()
+          ? caught.message
+          : "Очередь расчётов недоступна.");
+      });
+    return () => request.abort();
+  }, [revision, selectionKey]);
+
+  useEffect(() => () => requestRef.current?.abort(), []);
+
+  const latestJob = jobs[0] ?? null;
+  const activeJob = useMemo(
+    () => jobs.find((job) => OPEN_STATES.has(job.state)) ?? null,
+    [jobs],
+  );
+
+  useEffect(() => {
+    if (!latestJob || activeJob) return;
+    idempotencyKeys.current.delete(selectionKey);
+  }, [activeJob, latestJob, selectionKey]);
+
+  const refresh = useCallback(() => setRevision((value) => value + 1), []);
+
+  const submit = useCallback(async (): Promise<ObservatoryRecordedJob | null> => {
+    if (!sourceSessionId || !setupId || state === "submitting") return null;
+    if (activeJob) return activeJob;
+    requestRef.current?.abort();
+    const request = new AbortController();
+    requestRef.current = request;
+    const key = idempotencyKeys.current.get(selectionKey)
+      ?? createIdempotencyKey();
+    idempotencyKeys.current.set(selectionKey, key);
+    setState("submitting");
+    setError(null);
+    try {
+      const job = await submitObservatoryRecordedJob(sourceSessionId, setupId, key, {
+        signal: request.signal,
+      });
+      if (request.signal.aborted || requestRef.current !== request) return null;
+      setJobs((current) => [job, ...current.filter((candidate) => candidate.jobId !== job.jobId)]);
+      setState("ready");
+      return job;
+    } catch (caught) {
+      if (request.signal.aborted || requestRef.current !== request) return null;
+      setState("error");
+      setError(caught instanceof Error && caught.message.trim()
+        ? caught.message
+        : "Не удалось поставить расчёт в очередь.");
+      return null;
+    } finally {
+      if (requestRef.current === request) requestRef.current = null;
+    }
+  }, [activeJob, selectionKey, setupId, sourceSessionId, state]);
+
+  return { jobs, latestJob, activeJob, state, error, refresh, submit };
+}
+
+function createIdempotencyKey(): string {
+  const entropy = typeof globalThis.crypto?.randomUUID === "function"
+    ? globalThis.crypto.randomUUID()
+    : `${Date.now().toString(36)}-${Math.random().toString(16).slice(2)}`;
+  return `observatory-ui:${entropy}`;
+}
+
+export type ObservatoryRecordedJobsController = ReturnType<typeof useObservatoryRecordedJobs>;