feat: add RELLIS dataset compatibility smoke

apps/control-station/src/core/lidar/datasetGateway.ts

@@ -3,6 +3,57 @@ export type DatasetRepresentationId =
   | "normalized-scan"
   | "rolling-local-map";
 
+export type DatasetSourceAdmissionStatus =
+  | "blocked-storage-policy"
+  | "ready-for-download"
+  | "ready-for-smoke"
+  | "downloading"
+  | "downloaded"
+  | "verifying"
+  | "verified"
+  | "frame-ready"
+  | "smoke-ready";
+
+export interface DatasetSource {
+  sourceId: string;
+  sourceKind: "goose" | "rellis";
+  displayName: string;
+  role: string;
+  license: string;
+  commercialUse:
+    | "allowed-with-share-alike"
+    | "research-only-license-review-required";
+  format: string;
+  frameSemantics: "one-lidar-revolution";
+  sensor: string;
+  platforms: string[];
+  superclasses: string[];
+  smokeExampleMb: number | null;
+  primaryScanArchiveGb: number;
+  labelArchiveGb: number | null;
+  posesArchiveGb: number | null;
+  fragmentCount: number;
+  annotatedScanCount: number;
+  admissionStatus: DatasetSourceAdmissionStatus;
+  archive: {
+    bytesTransferred: number;
+    totalBytes: number;
+    sizeBytes: number | null;
+    sha256: string | null;
+    integrity: string;
+    vendorChecksumAvailable: false;
+  } | null;
+  frame: {
+    frameId: string;
+    pointCount: number;
+    semanticClassCount: number;
+    groundTruthGroundFraction: number;
+    previewPointCount: number;
+    previewSha256: string | null;
+    previewAvailable: true;
+  } | null;
+}
+
 export interface DatasetGatewayCatalog {
   storage: {
     configured: boolean;
@@ -13,42 +64,7 @@ export interface DatasetGatewayCatalog {
     requiredWindowsRoot: string;
     requiredWslRoot: string;
   };
-  source: {
-    sourceId: string;
-    displayName: string;
-    role: string;
-    license: string;
-    format: string;
-    frameSemantics: "one-lidar-revolution";
-    platforms: string[];
-    superclasses: string[];
-    validationArchiveGb: number;
-    admissionStatus:
-      | "blocked-storage-policy"
-      | "ready-for-download"
-      | "downloading"
-      | "downloaded"
-      | "verifying"
-      | "verified"
-      | "frame-ready";
-    archive: {
-      bytesTransferred: number;
-      totalBytes: number;
-      sizeBytes: number | null;
-      sha256: string | null;
-      integrity: string;
-      vendorChecksumAvailable: false;
-    } | null;
-    frame: {
-      frameId: string;
-      pointCount: number;
-      semanticClassCount: number;
-      groundTruthGroundFraction: number;
-      previewPointCount: number;
-      previewSha256: string;
-      previewAvailable: true;
-    } | null;
-  };
+  sources: DatasetSource[];
   representations: Array<{
     id: DatasetRepresentationId;
     title: string;
@@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
 }
 
 export interface DatasetNativeScanPreview {
-  sourceId: "goose-3d/v2025-08-22";
+  sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
   frameId: string;
   sourcePointCount: number;
   pointCount: number;
@@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
   semanticLabelIds: number[];
   semanticRgb0To255: number[];
   groundTruthGround: number[];
+  evaluationMask: number[];
   classes: Array<{
     labelId: number;
     className: string;
     hex: string;
-    challengeCategoryId: number;
-    challengeCategoryName: string;
+    challengeCategoryId: number | null;
+    challengeCategoryName: string | null;
+    groundTarget: "ground" | "non-ground" | "ignore" | null;
+    sourcePointCount: number | null;
   }>;
+  coordinateFrame: {
+    frameId: string;
+    x: "forward";
+    y: "left";
+    z: "up";
+  } | null;
+  compatibilitySmokeOnly: boolean;
 }
 
 export interface DatasetGroundMetrics {
@@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
 ): DatasetGatewayCatalog {
   const source = record(value, "Dataset Gateway");
   if (
-    source.schema_version !== "missioncore.dataset-gateway-catalog/v2"
+    source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
     || source.access !== "read-only"
   ) {
     throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
@@ -247,85 +273,158 @@ export function parseDatasetGatewayCatalog(
     throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
   }
   const sources = array(source.sources, "sources");
-  if (sources.length !== 1) {
-    throw new DatasetGatewayContractError("Ожидался один первичный dataset source");
+  if (sources.length !== 2) {
+    throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
   }
-  const dataset = record(sources[0], "sources[0]");
-  const download = record(dataset.download, "source.download");
-  const admission = record(dataset.admission, "source.admission");
-  if (download.automatic !== false) {
-    throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически");
-  }
-  const admissionStatus = admission.status;
+  const parsedSources = sources.map((value, index): DatasetSource => {
+    const dataset = record(value, `sources[${index}]`);
+    const sourceKind = string(dataset.source_kind, "source_kind", true);
+    const sourceId = string(dataset.source_id, "source_id", true);
+    if (
+      (sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
+      && (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
+    ) {
+      throw new DatasetGatewayContractError("Dataset source identity неизвестна");
+    }
+    if (dataset.frame_semantics !== "one-lidar-revolution") {
+      throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
+    }
+    const download = record(dataset.download, `sources[${index}].download`);
+    const statistics = record(dataset.statistics, `sources[${index}].statistics`);
+    const admission = record(dataset.admission, `sources[${index}].admission`);
+    if (download.automatic !== false) {
+      throw new DatasetGatewayContractError(
+        "Большой dataset нельзя загружать автоматически",
+      );
+    }
+    const admissionStatus = admission.status;
+    if (
+      admissionStatus !== "ready-for-download"
+      && admissionStatus !== "ready-for-smoke"
+      && admissionStatus !== "blocked-storage-policy"
+      && admissionStatus !== "downloading"
+      && admissionStatus !== "downloaded"
+      && admissionStatus !== "verifying"
+      && admissionStatus !== "verified"
+      && admissionStatus !== "frame-ready"
+      && admissionStatus !== "smoke-ready"
+    ) {
+      throw new DatasetGatewayContractError("source admission status неизвестен");
+    }
+    const archive = admission.archive === null
+      ? null
+      : record(admission.archive, `sources[${index}].admission.archive`);
+    const archiveValue = archive
+      ? {
+          bytesTransferred: number(
+            archive.bytes_transferred,
+            "archive.bytes_transferred",
+          ),
+          totalBytes: number(archive.total_bytes, "archive.total_bytes"),
+          sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
+          sha256: nullableDigest(archive.sha256, "archive.sha256"),
+          integrity: string(archive.integrity, "archive.integrity", true),
+          vendorChecksumAvailable: (() => {
+            if (archive.vendor_checksum_available !== false) {
+              throw new DatasetGatewayContractError(
+                "archive.vendor_checksum_available: ожидался false",
+              );
+            }
+            return false as const;
+          })(),
+        }
+      : null;
+    const admittedFrame = admission.frame === null
+      ? null
+      : record(admission.frame, `sources[${index}].admission.frame`);
+    const frameValue = admittedFrame
+      ? {
+          frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
+          pointCount: number(admittedFrame.point_count, "frame.point_count"),
+          semanticClassCount: number(
+            admittedFrame.semantic_class_count,
+            "frame.semantic_class_count",
+          ),
+          groundTruthGroundFraction: number(
+            admittedFrame.ground_truth_ground_fraction,
+            "frame.ground_truth_ground_fraction",
+          ),
+          previewPointCount: number(
+            admittedFrame.preview_point_count,
+            "frame.preview_point_count",
+          ),
+          previewSha256: nullableDigest(
+            admittedFrame.preview_sha256,
+            "frame.preview_sha256",
+          ),
+          previewAvailable: (() => {
+            if (admittedFrame.preview_available !== true) {
+              throw new DatasetGatewayContractError(
+                "frame.preview_available: ожидался true",
+              );
+            }
+            return true as const;
+          })(),
+        }
+      : null;
+    if (
+      sourceKind === "goose"
+      && frameValue
+      && frameValue.previewSha256 === null
+    ) {
+      throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
+    }
+    const commercialUse = dataset.commercial_use;
+    if (
+      commercialUse !== "allowed-with-share-alike"
+      && commercialUse !== "research-only-license-review-required"
+    ) {
+      throw new DatasetGatewayContractError("Dataset license scope неизвестен");
+    }
+    return {
+      sourceId,
+      sourceKind,
+      displayName: string(dataset.display_name, "display_name"),
+      role: string(dataset.role, "role", true),
+      license: string(dataset.license, "license"),
+      commercialUse,
+      format: string(dataset.format, "format", true),
+      frameSemantics: "one-lidar-revolution",
+      sensor: string(dataset.sensor, "sensor"),
+      platforms: displayStrings(dataset.platforms, "platforms"),
+      superclasses: strings(dataset.superclasses, "superclasses"),
+      smokeExampleMb: nullableNumber(
+        download.smoke_example_mb,
+        "smoke_example_mb",
+      ),
+      primaryScanArchiveGb: number(
+        download.primary_scan_archive_gb,
+        "primary_scan_archive_gb",
+      ),
+      labelArchiveGb: nullableNumber(
+        download.label_archive_gb,
+        "label_archive_gb",
+      ),
+      posesArchiveGb: nullableNumber(
+        download.poses_archive_gb,
+        "poses_archive_gb",
+      ),
+      fragmentCount: number(statistics.fragment_count, "fragment_count"),
+      annotatedScanCount: number(
+        statistics.annotated_scan_count,
+        "annotated_scan_count",
+      ),
+      admissionStatus,
+      archive: archiveValue,
+      frame: frameValue,
+    };
+  });
   if (
-    admissionStatus !== "ready-for-download"
-    && admissionStatus !== "blocked-storage-policy"
-    && admissionStatus !== "downloading"
-    && admissionStatus !== "downloaded"
-    && admissionStatus !== "verifying"
-    && admissionStatus !== "verified"
-    && admissionStatus !== "frame-ready"
+    parsedSources[0]?.sourceKind !== "goose"
+    || parsedSources[1]?.sourceKind !== "rellis"
   ) {
-    throw new DatasetGatewayContractError("source admission status неизвестен");
+    throw new DatasetGatewayContractError("Dataset source order несовместим");
   }
-  const archive = admission.archive === null
-    ? null
-    : record(admission.archive, "source.admission.archive");
-  const archiveValue = archive
-    ? {
-        bytesTransferred: number(
-          archive.bytes_transferred,
-          "archive.bytes_transferred",
-        ),
-        totalBytes: number(archive.total_bytes, "archive.total_bytes"),
-        sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
-        sha256: nullableDigest(archive.sha256, "archive.sha256"),
-        integrity: string(archive.integrity, "archive.integrity", true),
-        vendorChecksumAvailable: (() => {
-          if (archive.vendor_checksum_available !== false) {
-            throw new DatasetGatewayContractError(
-              "archive.vendor_checksum_available: ожидался false",
-            );
-          }
-          return false as const;
-        })(),
-      }
-    : null;
-  const admittedFrame = admission.frame === null
-    ? null
-    : record(admission.frame, "source.admission.frame");
-  const frameValue = admittedFrame
-    ? {
-        frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
-        pointCount: number(admittedFrame.point_count, "frame.point_count"),
-        semanticClassCount: number(
-          admittedFrame.semantic_class_count,
-          "frame.semantic_class_count",
-        ),
-        groundTruthGroundFraction: number(
-          admittedFrame.ground_truth_ground_fraction,
-          "frame.ground_truth_ground_fraction",
-        ),
-        previewPointCount: number(
-          admittedFrame.preview_point_count,
-          "frame.preview_point_count",
-        ),
-        previewSha256: nullableDigest(
-          admittedFrame.preview_sha256,
-          "frame.preview_sha256",
-        ) ?? (() => {
-          throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
-        })(),
-        previewAvailable: (() => {
-          if (admittedFrame.preview_available !== true) {
-            throw new DatasetGatewayContractError(
-              "frame.preview_available: ожидался true",
-            );
-          }
-          return true as const;
-        })(),
-      }
-    : null;
   const representations = array(
     source.representations,
     "representations",
@@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
   ) {
     throw new DatasetGatewayContractError("Vendor-map boundary завышен");
   }
-  if (dataset.frame_semantics !== "one-lidar-revolution") {
-    throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
-  }
   return {
     storage: {
       configured: boolean(storage.configured, "storage.configured"),
@@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
       ),
       requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
     },
-    source: {
-      sourceId: string(dataset.source_id, "source_id", true),
-      displayName: string(dataset.display_name, "display_name"),
-      role: string(dataset.role, "role", true),
-      license: string(dataset.license, "license"),
-      format: string(dataset.format, "format", true),
-      frameSemantics: "one-lidar-revolution",
-      platforms: displayStrings(dataset.platforms, "platforms"),
-      superclasses: strings(dataset.superclasses, "superclasses"),
-      validationArchiveGb: number(
-        download.validation_archive_gb,
-        "validation_archive_gb",
-      ),
-      admissionStatus,
-      archive: archiveValue,
-      frame: frameValue,
-    },
+    sources: parsedSources,
     representations,
     pipeline,
     currentInput: {
@@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
   value: unknown,
 ): DatasetNativeScanPreview {
   const source = record(value, "Dataset preview");
+  const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
+    && source.source_id === "goose-3d/v2025-08-22";
+  const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
+    && source.source_id === "rellis-3d/v1.1";
   if (
-    source.schema_version !== "missioncore.dataset-native-scan-preview/v1"
-    || source.source_id !== "goose-3d/v2025-08-22"
+    (!isGoose && !isRellis)
     || source.representation !== "native-scan"
     || source.sampling !== "deterministic-even-index"
   ) {
@@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
     "ground_truth_ground",
     1,
   );
+  const evaluationMask = isRellis
+    ? integers(source.evaluation_mask, "evaluation_mask", 1)
+    : Array.from({ length: pointCount }, () => 1);
   if (
     pointCount < 1
     || pointCount > 50_000
@@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
     || semanticLabelIds.length !== pointCount
     || semanticRgb0To255.length !== pointCount * 3
     || groundTruthGround.length !== pointCount
+    || evaluationMask.length !== pointCount
   ) {
     throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
   }
   const classes = array(source.classes, "classes").map((value, index) => {
     const item = record(value, `classes[${index}]`);
+    const groundTarget = item.ground_target;
+    if (
+      groundTarget !== undefined
+      && groundTarget !== "ground"
+      && groundTarget !== "non-ground"
+      && groundTarget !== "ignore"
+    ) {
+      throw new DatasetGatewayContractError("class.ground_target неизвестен");
+    }
+    const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
+      groundTarget === undefined ? null : groundTarget;
     return {
       labelId: number(item.label_id, "class.label_id"),
       className: string(item.class_name, "class.class_name", true),
       hex: string(item.hex, "class.hex"),
-      challengeCategoryId: number(
-        item.challenge_category_id,
-        "class.challenge_category_id",
-      ),
-      challengeCategoryName: string(
-        item.challenge_category_name,
-        "class.challenge_category_name",
-        true,
-      ),
+      challengeCategoryId: item.challenge_category_id === undefined
+        ? null
+        : number(item.challenge_category_id, "class.challenge_category_id"),
+      challengeCategoryName: item.challenge_category_name === undefined
+        ? null
+        : string(
+            item.challenge_category_name,
+            "class.challenge_category_name",
+            true,
+          ),
+      groundTarget: normalizedGroundTarget,
+      sourcePointCount: item.source_point_count === undefined
+        ? null
+        : number(item.source_point_count, "class.source_point_count"),
     };
   });
   const safety = record(source.safety, "safety");
   if (
     safety.visualization_only !== true
     || safety.navigation_or_safety_accepted !== false
+    || (isRellis && safety.compatibility_smoke_only !== true)
   ) {
     throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
   }
+  const coordinateFrame = isRellis
+    ? (() => {
+        const frame = record(source.coordinate_frame, "coordinate_frame");
+        if (
+          frame.frame_id !== "sensor/lidar/os1"
+          || frame.x !== "forward"
+          || frame.y !== "left"
+          || frame.z !== "up"
+          || frame.handedness !== "right"
+          || frame.transform_applied !== false
+        ) {
+          throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
+        }
+        return {
+          frameId: "sensor/lidar/os1",
+          x: "forward" as const,
+          y: "left" as const,
+          z: "up" as const,
+        };
+      })()
+    : null;
   return {
-    sourceId: "goose-3d/v2025-08-22",
+    sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
     frameId: string(source.frame_id, "frame_id", true),
     sourcePointCount,
     pointCount,
@@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
     semanticLabelIds,
     semanticRgb0To255,
     groundTruthGround,
+    evaluationMask,
     classes,
+    coordinateFrame,
+    compatibilitySmokeOnly: isRellis,
   };
 }
 
@@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
 }
 
 export async function fetchDatasetNativeScanPreview(
-  options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {},
+  options: {
+    sourceId?: DatasetNativeScanPreview["sourceId"];
+    signal?: AbortSignal;
+    fetcher?: DatasetFetch;
+  } = {},
 ): Promise<DatasetNativeScanPreview> {
   const fetcher = options.fetcher ?? fetch;
-  const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", {
+  const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
+  const response = await fetcher(
+    `/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
+    {
     method: "GET",
     headers: { Accept: "application/json" },
     signal: options.signal,
-  });
+    },
+  );
   return parseDatasetNativeScanPreview(await responseJson(response));
 }