feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
@@ -38,47 +38,57 @@ export function recordedObservationSources(
      spatialRegistration: "native",
    },
  };
-  const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => ({
-    id: source.id,
-    sourceId: source.id,
-    semanticChannelId: "camera.video.recorded",
-    label: source.label,
-    description: "Сохранённый видеоканал на общей временной шкале сессии",
-    modality: "video",
-    role: "auxiliary",
-    availability: "available",
-    transport: "recording",
-    endpointLabel: "Сохранённая сессия",
-    previewUrl: null,
-    delivery: {
-      id: `${launch.sessionId}:${source.id}`,
-      kind: "recorded-fmp4-manifest",
-      url: source.manifestUrl,
-      mediaType: source.mediaType,
-      manifestGenerationSha256: source.manifestGenerationSha256,
-      byteLength: source.byteLength,
-      timelineStartSeconds: source.timelineStartSeconds,
-      timelineEndSeconds: source.timelineEndSeconds,
-    },
-    activation: null,
-    provider: {
-      pluginId: "missioncore.session-archive",
-      pluginVersion: "1",
-      modelId: "recorded-media",
-      compatibilityProfileId: null,
-    },
-    binding: {},
-    capabilities: {
-      overlay: true,
-      fullscreen: true,
-      resizable: true,
-      defaultVisible: index < 2,
-      timelineMode: "recorded",
-      seekable: true,
-      sessionRecording: true,
-      clockId: "session_time",
-      spatialRegistration: "unresolved",
-    },
-  }));
+  const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => {
+    const perception = source.id.startsWith("recorded.perception.");
+    return {
+      id: source.id,
+      sourceId: source.id,
+      semanticChannelId: perception
+        ? "camera.perception.panoptic.recorded"
+        : "camera.video.recorded",
+      label: source.label,
+      description: perception
+        ? "Покадровая instance + semantic сегментация на общей временной шкале"
+        : "Сохранённый видеоканал на общей временной шкале сессии",
+      modality: "video",
+      role: "auxiliary",
+      availability: "available",
+      transport: "recording",
+      endpointLabel: "Сохранённая сессия",
+      previewUrl: null,
+      delivery: {
+        id: `${launch.sessionId}:${source.id}`,
+        kind: "recorded-fmp4-manifest",
+        url: source.manifestUrl,
+        mediaType: source.mediaType,
+        manifestGenerationSha256: source.manifestGenerationSha256,
+        byteLength: source.byteLength,
+        timelineStartSeconds: source.timelineStartSeconds,
+        timelineEndSeconds: source.timelineEndSeconds,
+      },
+      activation: null,
+      provider: {
+        pluginId: "missioncore.session-archive",
+        pluginVersion: "1",
+        modelId: perception ? "recorded-panoptic-perception" : "recorded-media",
+        compatibilityProfileId: null,
+      },
+      binding: {},
+      capabilities: {
+        overlay: true,
+        fullscreen: true,
+        resizable: true,
+        // AI presentation belongs to the unified Rerun composition. Keep the
+        // pre-rendered perception video as an explicit fallback instead of
+        // opening it as a second, independently controlled stream.
+        defaultVisible: !perception && index < 2,
+        timelineMode: "recorded",
+        seekable: true,
+        sessionRecording: true,
+        clockId: "session_time",
+        spatialRegistration: perception ? "calibrated" : "unresolved",
+      },
+    };
+  });
  return [spatial, ...media];
 }
@@ -1,12 +1,8 @@
-// A device-agnostic frontend safety policy. Sixteen channels covers multi-rig
-// vehicles while the independent byte/concurrency limits keep admission
-// bounded. One real accepted archive is ~163 MiB for a single camera, so the
-// old 128 MiB laboratory ceiling rejected a valid sealed generation before the
-// first manifest request. OPFS-backed sealed generations remain the scaling
-// path beyond this in-memory policy.
+// Camera count and preparation concurrency remain device-agnostic scheduling
+// policy. Recorded duration and aggregate bytes are deliberately not admission
+// criteria: sealed media is presented through a generation-bound HTTP stream,
+// so a one-, three- or ten-hour recording never has to fit in browser memory.
 export const MAX_RECORDED_CAMERA_SOURCES = 16;
-export const MAX_RECORDED_MEDIA_SOURCE_BYTES = 256 * 1024 * 1024;
-export const MAX_RECORDED_SESSION_CAMERA_BYTES = 512 * 1024 * 1024;
 export const MAX_CONCURRENT_RECORDED_CAMERA_PREPARATIONS = 1;

 export type RecordedAdmissionPhase = "loading" | "ready" | "error";
@@ -32,18 +28,12 @@ export function recordedCameraDescriptorPreflight(
 ): RecordedAdmissionPhase {
  if (sources.length > MAX_RECORDED_CAMERA_SOURCES) return "error";
  if (new Set(sources.map(({ id }) => id)).size !== sources.length) return "error";
-  let totalBytes = 0;
  for (const source of sources) {
    if (
      !source.id ||
      !Number.isSafeInteger(source.byteLength) ||
-      source.byteLength < 1 ||
-      source.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
+      source.byteLength < 1
    ) return "error";
-    totalBytes += source.byteLength;
-    if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) {
-      return "error";
-    }
  }
  return "ready";
 }
@@ -67,7 +57,6 @@ export function recordedSessionAdmissionPhase(
  if (sourceIds.length > MAX_RECORDED_CAMERA_SOURCES || spatialPhase === "error") {
    return "error";
  }
-  let totalBytes = 0;
  for (const sourceId of sourceIds) {
    const camera = cameras[sourceId];
    if (!camera || camera.phase === "error") return "error";
@@ -75,13 +64,10 @@ export function recordedSessionAdmissionPhase(
    if (camera.byteLength !== null) {
      if (
        !Number.isSafeInteger(camera.byteLength) ||
-        camera.byteLength < 1 ||
-        camera.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
+        camera.byteLength < 1
      ) return "error";
-      totalBytes += camera.byteLength;
    }
  }
-  if (totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) return "error";
  if (spatialPhase !== "ready") return "loading";
  return sourceIds.every((sourceId) => cameras[sourceId]?.phase === "ready")
    ? "ready"
@@ -1,7 +1,5 @@
 import {
  MAX_RECORDED_CAMERA_SOURCES,
-  MAX_RECORDED_MEDIA_SOURCE_BYTES,
-  MAX_RECORDED_SESSION_CAMERA_BYTES,
 } from "./recordedSessionAdmission";

 export type ObservationSessionStatus =
@@ -99,19 +97,8 @@ export interface ObservationRecordedMediaEpoch {
  timelineStartSeconds: number;
  timelineEndSeconds: number;
  mediaType: string;
-  initUrl: string;
-  initByteLength: number;
-  initSha256: string;
-  segmentCount: number;
-  segmentUrlPrefix: string;
-  segments: readonly ObservationRecordedMediaSegment[];
-}
-
-export interface ObservationRecordedMediaSegment {
-  sequence: number;
-  url: string;
  byteLength: number;
-  sha256: string;
+  streamUrl: string;
 }

 export interface ObservationRecordedMediaManifest {
@@ -215,18 +202,8 @@ const RECORDED_MEDIA_EPOCH_KEYS = new Set([
  "timeline_start_seconds",
  "timeline_end_seconds",
  "media_type",
-  "init_url",
-  "init_byte_length",
-  "init_sha256",
-  "segment_count",
-  "segment_url_prefix",
-  "segments",
-]);
-const RECORDED_MEDIA_SEGMENT_KEYS = new Set([
-  "sequence",
-  "url",
  "byte_length",
-  "sha256",
+  "stream_url",
 ]);
 const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
 const SAFE_MODALITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
@@ -234,10 +211,9 @@ const ISO_WITH_TIMEZONE = /^\d{4}-\d{2}-\d{2}T.+(?:Z|[+-]\d{2}:\d{2})$/;
 const SHA256 = /^[a-f0-9]{64}$/;
 const SAFE_RECORDING_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording\.rrd$/;
 const SAFE_PREPARATION_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording-preparation$/;
-const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/media\/[A-Za-z0-9._%-]+\/manifest$/;
+const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+|perception-media\/result-[a-f0-9]{64})\/manifest$/;
+const SAFE_MEDIA_STREAM_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+\/epochs\/[1-9][0-9]*|perception-media\/result-[a-f0-9]{64})\/recording\.mp4\?generation=[a-f0-9]{64}$/;
 const SAFE_MP4_MEDIA_TYPE = /^video\/mp4(?:; codecs="[A-Za-z0-9.,_-]+")?$/;
-const MAX_RECORDED_INIT_BYTES = 8 * 1024 * 1024;
-const MAX_RECORDED_SEGMENT_BYTES = 64 * 1024 * 1024;

 export class ObservationSessionContractError extends Error {
  constructor(message: string) {
@@ -570,15 +546,6 @@ export function decodeObservationSessionReplay(
      "Descriptor записи содержит повторяющиеся медиаканалы.",
    );
  }
-  const mediaByteLength = mediaSources.reduce((total, source) => total + source.byteLength, 0);
-  if (
-    !Number.isSafeInteger(mediaByteLength) ||
-    mediaByteLength > MAX_RECORDED_SESSION_CAMERA_BYTES
-  ) {
-    throw new ObservationSessionContractError(
-      "Совокупный объём записанных медиаканалов превышает безопасный лимит браузера.",
-    );
-  }
  return {
    kind: "rerun-recording",
    sessionId,
@@ -709,7 +676,10 @@ function decodeRecordedMediaSource(
  }
  assertExactKeys(value, RECORDED_MEDIA_SOURCE_KEYS, `Медиаканал ${index}`);
  const id = requireString(value.id, `media_sources[${index}].id`, 128);
-  if (!SAFE_ID.test(id) || !id.startsWith("recorded.camera.")) {
+  if (
+    !SAFE_ID.test(id) ||
+    (!id.startsWith("recorded.camera.") && !id.startsWith("recorded.perception."))
+  ) {
    throw new ObservationSessionContractError("Медиаканал содержит небезопасный opaque id.");
  }
  if (value.modality !== "video" || value.media_type !== "video/mp4") {
@@ -728,10 +698,10 @@ function decodeRecordedMediaSource(
      "Медиаканал не содержит immutable generation SHA-256.",
    );
  }
-  const sessionPrefix = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/media/`;
+  const sessionBase = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/`;
  if (
    !SAFE_MEDIA_MANIFEST_URL.test(manifestUrl) ||
-    !manifestUrl.startsWith(sessionPrefix) ||
+    !manifestUrl.startsWith(sessionBase) ||
    manifestUrl.includes("..")
  ) {
    throw new ObservationSessionContractError(
@@ -765,7 +735,7 @@ function decodeRecordedMediaSource(
    byteLength: requireFiniteNumber(
      value.byte_length,
      `media_sources[${index}].byte_length`,
-      { minimum: 1, maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES, integer: true },
+      { minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
    ),
    mediaType: "video/mp4",
    timelineStartSeconds,
@@ -784,22 +754,21 @@ export function decodeObservationRecordedMediaManifest(
  }
  assertExactKeys(payload, RECORDED_MEDIA_MANIFEST_KEYS, "Manifest записанного видео");
  if (
-    payload.schema_version !== "missioncore.observation-recorded-media/v2" ||
+    payload.schema_version !== "missioncore.observation-recorded-media/v3" ||
    payload.source_id !== source.id ||
    typeof payload.generation_sha256 !== "string" ||
    !SHA256.test(payload.generation_sha256) ||
    payload.generation_sha256 !== source.manifestGenerationSha256 ||
    payload.synchronization !== "host-arrival-best-effort" ||
    !Array.isArray(payload.epochs) ||
-    payload.epochs.length < 1 ||
-    payload.epochs.length > 1_000
+    payload.epochs.length < 1
  ) {
    throw new ObservationSessionContractError("Manifest записанного видео несовместим.");
  }
  const manifestBase = source.manifestUrl.slice(0, -"/manifest".length);
  const manifestByteLength = requireFiniteNumber(payload.byte_length, "manifest.byte_length", {
    minimum: 1,
-    maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES,
+    maximum: Number.MAX_SAFE_INTEGER,
    integer: true,
  });
  if (manifestByteLength !== source.byteLength) {
@@ -833,7 +802,7 @@ export function decodeObservationRecordedMediaManifest(
    assertExactKeys(entry, RECORDED_MEDIA_EPOCH_KEYS, `Codec epoch ${index}`);
    const ordinal = requireFiniteNumber(entry.ordinal, `epochs[${index}].ordinal`, {
      minimum: 1,
-      maximum: 1_000,
+      maximum: Number.MAX_SAFE_INTEGER,
      integer: true,
    });
    if (ordinal !== index + 1) {
@@ -860,87 +829,28 @@ export function decodeObservationRecordedMediaManifest(
    if (!SAFE_MP4_MEDIA_TYPE.test(mediaType)) {
      throw new ObservationSessionContractError("Codec epoch содержит небезопасный media type.");
    }
-    const initUrl = requireString(entry.init_url, `epochs[${index}].init_url`, 512);
-    const segmentUrlPrefix = requireString(
-      entry.segment_url_prefix,
-      `epochs[${index}].segment_url_prefix`,
-      512,
+    const byteLength = requireFiniteNumber(
+      entry.byte_length,
+      `epochs[${index}].byte_length`,
+      { minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
    );
-    const expectedEpochBase = `${manifestBase}/epochs/${ordinal}`;
+    const streamUrl = requireString(entry.stream_url, `epochs[${index}].stream_url`, 768);
+    const expectedEpochBase = source.id.startsWith("recorded.perception.")
+      ? manifestBase
+      : `${manifestBase}/epochs/${ordinal}`;
+    const expectedStreamUrl =
+      `${expectedEpochBase}/recording.mp4?generation=${payload.generation_sha256}`;
    if (
-      initUrl !== `${expectedEpochBase}/init.mp4` ||
-      segmentUrlPrefix !== `${expectedEpochBase}/segments/` ||
-      initUrl.includes("..") ||
-      segmentUrlPrefix.includes("..")
+      !SAFE_MEDIA_STREAM_URL.test(streamUrl) ||
+      streamUrl !== expectedStreamUrl ||
+      streamUrl.includes("..")
    ) {
      throw new ObservationSessionContractError("Codec epoch содержит небезопасный API URL.");
    }
-    const initByteLength = requireFiniteNumber(
-      entry.init_byte_length,
-      `epochs[${index}].init_byte_length`,
-      { minimum: 1, maximum: MAX_RECORDED_INIT_BYTES, integer: true },
-    );
-    if (typeof entry.init_sha256 !== "string" || !SHA256.test(entry.init_sha256)) {
-      throw new ObservationSessionContractError("Codec epoch не содержит SHA-256 init-сегмента.");
-    }
-    const segmentCount = requireFiniteNumber(
-      entry.segment_count,
-      `epochs[${index}].segment_count`,
-      { minimum: 1, maximum: 500_000, integer: true },
-    );
-    if (!Array.isArray(entry.segments) || entry.segments.length !== segmentCount) {
+    declaredBytes += byteLength;
+    if (!Number.isSafeInteger(declaredBytes)) {
      throw new ObservationSessionContractError(
-        "Codec epoch содержит неполный список канонических сегментов.",
-      );
-    }
-    const segments = entry.segments.map((segment, segmentIndex): ObservationRecordedMediaSegment => {
-      if (!isRecord(segment)) {
-        throw new ObservationSessionContractError(
-          `Сегмент epochs[${index}].segments[${segmentIndex}] должен быть объектом.`,
-        );
-      }
-      assertExactKeys(
-        segment,
-        RECORDED_MEDIA_SEGMENT_KEYS,
-        `Сегмент epochs[${index}].segments[${segmentIndex}]`,
-      );
-      const sequence = requireFiniteNumber(
-        segment.sequence,
-        `epochs[${index}].segments[${segmentIndex}].sequence`,
-        { minimum: 1, maximum: 500_000, integer: true },
-      );
-      if (sequence !== segmentIndex + 1) {
-        throw new ObservationSessionContractError(
-          "Codec epoch содержит непоследовательный сегмент.",
-        );
-      }
-      const url = requireString(
-        segment.url,
-        `epochs[${index}].segments[${segmentIndex}].url`,
-        512,
-      );
-      if (url !== `${segmentUrlPrefix}${sequence}.m4s` || url.includes("..")) {
-        throw new ObservationSessionContractError(
-          "Codec epoch содержит небезопасный URL сегмента.",
-        );
-      }
-      const byteLength = requireFiniteNumber(
-        segment.byte_length,
-        `epochs[${index}].segments[${segmentIndex}].byte_length`,
-        { minimum: 1, maximum: MAX_RECORDED_SEGMENT_BYTES, integer: true },
-      );
-      if (typeof segment.sha256 !== "string" || !SHA256.test(segment.sha256)) {
-        throw new ObservationSessionContractError(
-          "Codec epoch содержит сегмент без SHA-256.",
-        );
-      }
-      declaredBytes += byteLength;
-      return { sequence, url, byteLength, sha256: segment.sha256 };
-    });
-    declaredBytes += initByteLength;
-    if (declaredBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES) {
-      throw new ObservationSessionContractError(
-        "Записанный медиаканал превышает безопасный лимит браузера.",
+        "Суммарный размер codec epoch выходит за точный числовой диапазон клиента.",
      );
    }
    return {
@@ -948,17 +858,13 @@ export function decodeObservationRecordedMediaManifest(
      timelineStartSeconds,
      timelineEndSeconds,
      mediaType,
-      initUrl,
-      initByteLength,
-      initSha256: entry.init_sha256,
-      segmentCount,
-      segmentUrlPrefix,
-      segments,
+      byteLength,
+      streamUrl,
    };
  });
  if (declaredBytes !== manifestByteLength) {
    throw new ObservationSessionContractError(
-      "Сумма init- и media-сегментов не совпадает с размером immutable manifest.",
+      "Сумма codec epoch не совпадает с размером immutable manifest.",
    );
  }
  if (
@@ -285,11 +285,16 @@ export async function resolveObservationSessionReplay(
  sessionId: string,
  options: ObservationPreparationPollingOptions,
 ): Promise<ObservationSessionReplayLaunch> {
-  const response = await withRequestTimeout(
-    options.signal,
-    Math.max(100, options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS),
-    (signal) => replayObservationSession(sessionId, { signal, fetcher: options.fetcher }),
-  );
+  // The initial replay request may restore and integrity-check large, already
+  // published artifacts before it can return either a launch descriptor or a
+  // background-preparation handle. Its duration therefore scales with the
+  // recording package and must not be confused with a stalled status poll.
+  // Keep it cancellable by the owning UI attempt, but do not impose the short
+  // per-poll timeout used once the server has returned a preparation handle.
+  const response = await replayObservationSession(sessionId, {
+    signal: options.signal,
+    fetcher: options.fetcher,
+  });
  if (response.kind === "ready") return response.launch;
  if (!pendingPreparation(response.preparation)) {
    options.onUpdate?.(response.preparation);
@@ -22,6 +22,9 @@ export interface ViewerSettings {
  show_points: boolean;
  show_trajectory: boolean;
  show_grid: boolean;
+  show_detections_2d: boolean;
+  show_segmentation: boolean;
+  show_cuboids_3d: boolean;
 }

 export interface StreamMetrics {
@@ -29,6 +32,10 @@ export interface StreamMetrics {
  frameRateHz?: number | null;
  pointCount?: number | null;
  droppedPreviewFrames?: number | null;
+  aiLatencyMs?: number | null;
+  aiFrameRateHz?: number | null;
+  aiDroppedFrames?: number | null;
+  aiStaleMs?: number | null;
  elapsedSeconds?: number | null;
  routeDistanceMeters?: number | null;
  speedMetersPerSecond?: number | null;