feat: add RELLIS dataset compatibility smoke

This commit is contained in:
DCCONSTRUCTIONS
2026-07-25 20:45:30 +03:00
parent 13c35ff446
commit 054feec0d2
19 changed files with 1633 additions and 277 deletions
@@ -3,6 +3,57 @@ export type DatasetRepresentationId =
| "normalized-scan"
| "rolling-local-map";
export type DatasetSourceAdmissionStatus =
| "blocked-storage-policy"
| "ready-for-download"
| "ready-for-smoke"
| "downloading"
| "downloaded"
| "verifying"
| "verified"
| "frame-ready"
| "smoke-ready";
export interface DatasetSource {
sourceId: string;
sourceKind: "goose" | "rellis";
displayName: string;
role: string;
license: string;
commercialUse:
| "allowed-with-share-alike"
| "research-only-license-review-required";
format: string;
frameSemantics: "one-lidar-revolution";
sensor: string;
platforms: string[];
superclasses: string[];
smokeExampleMb: number | null;
primaryScanArchiveGb: number;
labelArchiveGb: number | null;
posesArchiveGb: number | null;
fragmentCount: number;
annotatedScanCount: number;
admissionStatus: DatasetSourceAdmissionStatus;
archive: {
bytesTransferred: number;
totalBytes: number;
sizeBytes: number | null;
sha256: string | null;
integrity: string;
vendorChecksumAvailable: false;
} | null;
frame: {
frameId: string;
pointCount: number;
semanticClassCount: number;
groundTruthGroundFraction: number;
previewPointCount: number;
previewSha256: string | null;
previewAvailable: true;
} | null;
}
export interface DatasetGatewayCatalog {
storage: {
configured: boolean;
@@ -13,42 +64,7 @@ export interface DatasetGatewayCatalog {
requiredWindowsRoot: string;
requiredWslRoot: string;
};
source: {
sourceId: string;
displayName: string;
role: string;
license: string;
format: string;
frameSemantics: "one-lidar-revolution";
platforms: string[];
superclasses: string[];
validationArchiveGb: number;
admissionStatus:
| "blocked-storage-policy"
| "ready-for-download"
| "downloading"
| "downloaded"
| "verifying"
| "verified"
| "frame-ready";
archive: {
bytesTransferred: number;
totalBytes: number;
sizeBytes: number | null;
sha256: string | null;
integrity: string;
vendorChecksumAvailable: false;
} | null;
frame: {
frameId: string;
pointCount: number;
semanticClassCount: number;
groundTruthGroundFraction: number;
previewPointCount: number;
previewSha256: string;
previewAvailable: true;
} | null;
};
sources: DatasetSource[];
representations: Array<{
id: DatasetRepresentationId;
title: string;
@@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
}
export interface DatasetNativeScanPreview {
sourceId: "goose-3d/v2025-08-22";
sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
frameId: string;
sourcePointCount: number;
pointCount: number;
@@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
semanticLabelIds: number[];
semanticRgb0To255: number[];
groundTruthGround: number[];
evaluationMask: number[];
classes: Array<{
labelId: number;
className: string;
hex: string;
challengeCategoryId: number;
challengeCategoryName: string;
challengeCategoryId: number | null;
challengeCategoryName: string | null;
groundTarget: "ground" | "non-ground" | "ignore" | null;
sourcePointCount: number | null;
}>;
coordinateFrame: {
frameId: string;
x: "forward";
y: "left";
z: "up";
} | null;
compatibilitySmokeOnly: boolean;
}
export interface DatasetGroundMetrics {
@@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
): DatasetGatewayCatalog {
const source = record(value, "Dataset Gateway");
if (
source.schema_version !== "missioncore.dataset-gateway-catalog/v2"
source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
|| source.access !== "read-only"
) {
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
@@ -247,85 +273,158 @@ export function parseDatasetGatewayCatalog(
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
}
const sources = array(source.sources, "sources");
if (sources.length !== 1) {
throw new DatasetGatewayContractError("Ожидался один первичный dataset source");
if (sources.length !== 2) {
throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
}
const dataset = record(sources[0], "sources[0]");
const download = record(dataset.download, "source.download");
const admission = record(dataset.admission, "source.admission");
if (download.automatic !== false) {
throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически");
}
const admissionStatus = admission.status;
const parsedSources = sources.map((value, index): DatasetSource => {
const dataset = record(value, `sources[${index}]`);
const sourceKind = string(dataset.source_kind, "source_kind", true);
const sourceId = string(dataset.source_id, "source_id", true);
if (
(sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
&& (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
) {
throw new DatasetGatewayContractError("Dataset source identity неизвестна");
}
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
}
const download = record(dataset.download, `sources[${index}].download`);
const statistics = record(dataset.statistics, `sources[${index}].statistics`);
const admission = record(dataset.admission, `sources[${index}].admission`);
if (download.automatic !== false) {
throw new DatasetGatewayContractError(
"Большой dataset нельзя загружать автоматически",
);
}
const admissionStatus = admission.status;
if (
admissionStatus !== "ready-for-download"
&& admissionStatus !== "ready-for-smoke"
&& admissionStatus !== "blocked-storage-policy"
&& admissionStatus !== "downloading"
&& admissionStatus !== "downloaded"
&& admissionStatus !== "verifying"
&& admissionStatus !== "verified"
&& admissionStatus !== "frame-ready"
&& admissionStatus !== "smoke-ready"
) {
throw new DatasetGatewayContractError("source admission status неизвестен");
}
const archive = admission.archive === null
? null
: record(admission.archive, `sources[${index}].admission.archive`);
const archiveValue = archive
? {
bytesTransferred: number(
archive.bytes_transferred,
"archive.bytes_transferred",
),
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
sha256: nullableDigest(archive.sha256, "archive.sha256"),
integrity: string(archive.integrity, "archive.integrity", true),
vendorChecksumAvailable: (() => {
if (archive.vendor_checksum_available !== false) {
throw new DatasetGatewayContractError(
"archive.vendor_checksum_available: ожидался false",
);
}
return false as const;
})(),
}
: null;
const admittedFrame = admission.frame === null
? null
: record(admission.frame, `sources[${index}].admission.frame`);
const frameValue = admittedFrame
? {
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
pointCount: number(admittedFrame.point_count, "frame.point_count"),
semanticClassCount: number(
admittedFrame.semantic_class_count,
"frame.semantic_class_count",
),
groundTruthGroundFraction: number(
admittedFrame.ground_truth_ground_fraction,
"frame.ground_truth_ground_fraction",
),
previewPointCount: number(
admittedFrame.preview_point_count,
"frame.preview_point_count",
),
previewSha256: nullableDigest(
admittedFrame.preview_sha256,
"frame.preview_sha256",
),
previewAvailable: (() => {
if (admittedFrame.preview_available !== true) {
throw new DatasetGatewayContractError(
"frame.preview_available: ожидался true",
);
}
return true as const;
})(),
}
: null;
if (
sourceKind === "goose"
&& frameValue
&& frameValue.previewSha256 === null
) {
throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
}
const commercialUse = dataset.commercial_use;
if (
commercialUse !== "allowed-with-share-alike"
&& commercialUse !== "research-only-license-review-required"
) {
throw new DatasetGatewayContractError("Dataset license scope неизвестен");
}
return {
sourceId,
sourceKind,
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
commercialUse,
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
sensor: string(dataset.sensor, "sensor"),
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
smokeExampleMb: nullableNumber(
download.smoke_example_mb,
"smoke_example_mb",
),
primaryScanArchiveGb: number(
download.primary_scan_archive_gb,
"primary_scan_archive_gb",
),
labelArchiveGb: nullableNumber(
download.label_archive_gb,
"label_archive_gb",
),
posesArchiveGb: nullableNumber(
download.poses_archive_gb,
"poses_archive_gb",
),
fragmentCount: number(statistics.fragment_count, "fragment_count"),
annotatedScanCount: number(
statistics.annotated_scan_count,
"annotated_scan_count",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
};
});
if (
admissionStatus !== "ready-for-download"
&& admissionStatus !== "blocked-storage-policy"
&& admissionStatus !== "downloading"
&& admissionStatus !== "downloaded"
&& admissionStatus !== "verifying"
&& admissionStatus !== "verified"
&& admissionStatus !== "frame-ready"
parsedSources[0]?.sourceKind !== "goose"
|| parsedSources[1]?.sourceKind !== "rellis"
) {
throw new DatasetGatewayContractError("source admission status неизвестен");
throw new DatasetGatewayContractError("Dataset source order несовместим");
}
const archive = admission.archive === null
? null
: record(admission.archive, "source.admission.archive");
const archiveValue = archive
? {
bytesTransferred: number(
archive.bytes_transferred,
"archive.bytes_transferred",
),
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
sha256: nullableDigest(archive.sha256, "archive.sha256"),
integrity: string(archive.integrity, "archive.integrity", true),
vendorChecksumAvailable: (() => {
if (archive.vendor_checksum_available !== false) {
throw new DatasetGatewayContractError(
"archive.vendor_checksum_available: ожидался false",
);
}
return false as const;
})(),
}
: null;
const admittedFrame = admission.frame === null
? null
: record(admission.frame, "source.admission.frame");
const frameValue = admittedFrame
? {
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
pointCount: number(admittedFrame.point_count, "frame.point_count"),
semanticClassCount: number(
admittedFrame.semantic_class_count,
"frame.semantic_class_count",
),
groundTruthGroundFraction: number(
admittedFrame.ground_truth_ground_fraction,
"frame.ground_truth_ground_fraction",
),
previewPointCount: number(
admittedFrame.preview_point_count,
"frame.preview_point_count",
),
previewSha256: nullableDigest(
admittedFrame.preview_sha256,
"frame.preview_sha256",
) ?? (() => {
throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
})(),
previewAvailable: (() => {
if (admittedFrame.preview_available !== true) {
throw new DatasetGatewayContractError(
"frame.preview_available: ожидался true",
);
}
return true as const;
})(),
}
: null;
const representations = array(
source.representations,
"representations",
@@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
) {
throw new DatasetGatewayContractError("Vendor-map boundary завышен");
}
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
}
return {
storage: {
configured: boolean(storage.configured, "storage.configured"),
@@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
),
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
},
source: {
sourceId: string(dataset.source_id, "source_id", true),
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
validationArchiveGb: number(
download.validation_archive_gb,
"validation_archive_gb",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
},
sources: parsedSources,
representations,
pipeline,
currentInput: {
@@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
value: unknown,
): DatasetNativeScanPreview {
const source = record(value, "Dataset preview");
const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
&& source.source_id === "goose-3d/v2025-08-22";
const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
&& source.source_id === "rellis-3d/v1.1";
if (
source.schema_version !== "missioncore.dataset-native-scan-preview/v1"
|| source.source_id !== "goose-3d/v2025-08-22"
(!isGoose && !isRellis)
|| source.representation !== "native-scan"
|| source.sampling !== "deterministic-even-index"
) {
@@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
"ground_truth_ground",
1,
);
const evaluationMask = isRellis
? integers(source.evaluation_mask, "evaluation_mask", 1)
: Array.from({ length: pointCount }, () => 1);
if (
pointCount < 1
|| pointCount > 50_000
@@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
|| semanticLabelIds.length !== pointCount
|| semanticRgb0To255.length !== pointCount * 3
|| groundTruthGround.length !== pointCount
|| evaluationMask.length !== pointCount
) {
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
}
const classes = array(source.classes, "classes").map((value, index) => {
const item = record(value, `classes[${index}]`);
const groundTarget = item.ground_target;
if (
groundTarget !== undefined
&& groundTarget !== "ground"
&& groundTarget !== "non-ground"
&& groundTarget !== "ignore"
) {
throw new DatasetGatewayContractError("class.ground_target неизвестен");
}
const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
groundTarget === undefined ? null : groundTarget;
return {
labelId: number(item.label_id, "class.label_id"),
className: string(item.class_name, "class.class_name", true),
hex: string(item.hex, "class.hex"),
challengeCategoryId: number(
item.challenge_category_id,
"class.challenge_category_id",
),
challengeCategoryName: string(
item.challenge_category_name,
"class.challenge_category_name",
true,
),
challengeCategoryId: item.challenge_category_id === undefined
? null
: number(item.challenge_category_id, "class.challenge_category_id"),
challengeCategoryName: item.challenge_category_name === undefined
? null
: string(
item.challenge_category_name,
"class.challenge_category_name",
true,
),
groundTarget: normalizedGroundTarget,
sourcePointCount: item.source_point_count === undefined
? null
: number(item.source_point_count, "class.source_point_count"),
};
});
const safety = record(source.safety, "safety");
if (
safety.visualization_only !== true
|| safety.navigation_or_safety_accepted !== false
|| (isRellis && safety.compatibility_smoke_only !== true)
) {
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
}
const coordinateFrame = isRellis
? (() => {
const frame = record(source.coordinate_frame, "coordinate_frame");
if (
frame.frame_id !== "sensor/lidar/os1"
|| frame.x !== "forward"
|| frame.y !== "left"
|| frame.z !== "up"
|| frame.handedness !== "right"
|| frame.transform_applied !== false
) {
throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
}
return {
frameId: "sensor/lidar/os1",
x: "forward" as const,
y: "left" as const,
z: "up" as const,
};
})()
: null;
return {
sourceId: "goose-3d/v2025-08-22",
sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
frameId: string(source.frame_id, "frame_id", true),
sourcePointCount,
pointCount,
@@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
semanticLabelIds,
semanticRgb0To255,
groundTruthGround,
evaluationMask,
classes,
coordinateFrame,
compatibilitySmokeOnly: isRellis,
};
}
@@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
}
export async function fetchDatasetNativeScanPreview(
options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {},
options: {
sourceId?: DatasetNativeScanPreview["sourceId"];
signal?: AbortSignal;
fetcher?: DatasetFetch;
} = {},
): Promise<DatasetNativeScanPreview> {
const fetcher = options.fetcher ?? fetch;
const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", {
const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
const response = await fetcher(
`/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
{
method: "GET",
headers: { Accept: "application/json" },
signal: options.signal,
});
},
);
return parseDatasetNativeScanPreview(await responseJson(response));
}
@@ -224,7 +224,9 @@
}
.dataset-fragment-comparison,
.dataset-fragment-risks {
.dataset-fragment-risks,
.rellis-review__metrics,
.rellis-review__policy > div {
grid-template-columns: 1fr;
}
@@ -1223,6 +1223,85 @@
line-height: 1.5;
}
.rellis-review__metrics {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__metrics article {
display: grid;
gap: 0.2rem;
border-radius: 0.85rem;
background: var(--station-panel);
padding: 0.78rem 0.85rem;
}
.rellis-review__metrics span,
.rellis-review__metrics small {
color: var(--nodedc-text-muted);
font-size: 0.55rem;
}
.rellis-review__metrics strong {
color: var(--nodedc-text-primary);
font-size: 1.05rem;
}
.rellis-review__policy {
display: grid;
gap: 0.7rem;
border-radius: 1rem;
background: var(--station-panel);
padding: 0.9rem 1rem;
}
.rellis-review__policy h3,
.rellis-review__policy p {
margin: 0;
}
.rellis-review__policy h3 {
margin-top: 0.28rem;
color: var(--nodedc-text-primary);
font-size: 0.95rem;
}
.rellis-review__policy p {
max-width: 52rem;
margin-top: 0.3rem;
color: var(--nodedc-text-muted);
font-size: 0.6rem;
line-height: 1.5;
}
.rellis-review__policy > div {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__policy article {
display: grid;
align-content: start;
gap: 0.35rem;
border-radius: 0.75rem;
background: rgb(255 255 255 / 0.03);
padding: 0.7rem 0.75rem;
}
.rellis-review__policy article span {
color: var(--nodedc-text-muted);
font-size: 0.54rem;
}
.rellis-review__policy article strong {
color: var(--nodedc-text-secondary);
font-size: 0.62rem;
font-weight: 560;
line-height: 1.5;
}
.dataset-sequence-timeline {
height: 2rem;
}
@@ -2043,6 +2122,11 @@
font-size: 1rem;
}
.dataset-library__entries {
display: grid;
gap: 1rem;
}
.dataset-entry {
display: grid;
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);
@@ -8,12 +8,15 @@ import {
import {
fetchDatasetGatewayCatalog,
type DatasetGatewayCatalog,
type DatasetSource,
type DatasetSourceAdmissionStatus,
} from "../core/lidar/datasetGateway";
import {
fetchPolygonRunCatalog,
type PolygonRunCatalog,
} from "../core/polygon/runArchive";
import { GooseDatasetReview } from "./GooseDatasetReview";
import { RellisDatasetReview } from "./RellisDatasetReview";
const representationLabels: Record<string, string> = {
"native-scan": "Исходный скан",
@@ -27,15 +30,17 @@ function errorMessage(error: unknown): string {
: "Каталог датасетов недоступен.";
}
function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) {
const labels: Record<typeof status, string> = {
function admissionLabel(status: DatasetSourceAdmissionStatus): string {
const labels: Record<DatasetSourceAdmissionStatus, string> = {
"blocked-storage-policy": "Worker не настроен",
"ready-for-download": "Готов к загрузке",
"ready-for-smoke": "Готов к smoke",
downloading: "Загружается",
downloaded: "Загружен",
verifying: "Проверяется",
verified: "Проверен",
"frame-ready": "Первый кадр готов",
"frame-ready": "Dataset готов",
"smoke-ready": "Smoke готов",
};
return labels[status];
}
@@ -55,12 +60,42 @@ function isGooseQualificationRun(
&& item.scenarioGeneration.startsWith("goose-3d");
}
function sourceReady(source: DatasetSource): boolean {
return source.admissionStatus === "frame-ready"
|| source.admissionStatus === "smoke-ready";
}
function sourceDescription(source: DatasetSource): string {
if (source.sourceKind === "goose") {
return "Восемь фрагментов и 961 разреженный размеченный оборот VLS-128. Здесь уже опубликовано покадровое сравнение текущего ground baseline и Patchwork++.";
}
return "Независимый off-road источник с Ouster OS1 64. Сейчас открыт официальный frame 000104 для проверки reader, осей, ontology и ground-policy; полный benchmark ещё не выполнялся.";
}
function sourceFooter(source: DatasetSource, storageReady: boolean): string {
if (source.admissionStatus === "frame-ready") {
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
}
if (source.admissionStatus === "smoke-ready") {
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
}
if (source.admissionStatus === "downloading") {
return "Архив остаётся на D worker; после загрузки проверяются digest, лицензия и point-label alignment.";
}
if (!storageReady) {
return "Сначала нужен допущенный Dataset Root на диске D worker.";
}
return source.sourceKind === "rellis"
? "Следующий шаг: пропустить официальный 24 МБ example через smoke admission."
: "Следующий шаг: загрузить validation archive на worker и проверить hash/license.";
}
export function DatasetGatewayWorkspace() {
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [reloadGeneration, setReloadGeneration] = useState(0);
const [previewOpen, setPreviewOpen] = useState(false);
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
const [runError, setRunError] = useState<string | null>(null);
@@ -85,7 +120,9 @@ export function DatasetGatewayWorkspace() {
}, [reloadGeneration]);
useEffect(() => {
if (catalog?.source.admissionStatus !== "downloading") return;
if (!catalog?.sources.some((source) => source.admissionStatus === "downloading")) {
return;
}
const timer = window.setTimeout(
() => setReloadGeneration((value) => value + 1),
5_000,
@@ -113,10 +150,10 @@ export function DatasetGatewayWorkspace() {
return () => controller.abort();
}, []);
const downloadProgress = catalog?.source.archive
? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes
: null;
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
const openSource = catalog?.sources.find(
(source) => source.sourceId === openSourceId,
) ?? null;
return (
<div className="standard-workspace dataset-workspace">
@@ -125,17 +162,17 @@ export function DatasetGatewayWorkspace() {
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
<h2>Независимая проверка алгоритмов</h2>
<p>
Датасет содержит известные размеченные данные. Мы показываем
конкретный фрагмент, результаты каждого алгоритма и ошибки
относительно разметки. Он не подмешивается в диагностику реального
сенсора и не заменяет closed-loop симуляцию.
Каждый источник остаётся отдельным: собственные sensor domain,
разметка, лицензия и результаты. Совпадение результата на GOOSE и
RELLIS снижает риск подгонки под один датасет, но не заменяет replay
реального сенсора и closed-loop симуляцию.
</p>
</div>
<ol aria-label="Как использовать датасет">
<li><span>01</span>Выбрать фрагмент</li>
<li><span>02</span>Посмотреть сканы</li>
<li><span>01</span>Выбрать источник</li>
<li><span>02</span>Проверить scan и labels</li>
<li><span>03</span>Сравнить алгоритмы</li>
<li><span>04</span>Проверить риски</li>
<li><span>04</span>Проверить переносимость</li>
</ol>
</section>
@@ -171,80 +208,91 @@ export function DatasetGatewayWorkspace() {
</StatusBadge>
</header>
<article className="dataset-entry">
<div className="dataset-entry__identity">
<span className="dataset-entry__mark">G</span>
<div>
<span>PUBLIC · OFF-ROAD · LABELED</span>
<h3>{catalog.source.displayName}</h3>
<p>
Validation split: восемь фрагментов исходных записей и 961
разреженный размеченный оборот VLS-128. Это коллекция native
scans для perception, а не непрерывное видео или траектория
машины.
</p>
</div>
</div>
<dl>
<div>
<dt>Состояние</dt>
<dd>{admissionLabel(catalog.source.admissionStatus)}</dd>
</div>
<div>
<dt>Validation</dt>
<dd>{catalog.source.validationArchiveGb} ГБ</dd>
</div>
<div>
<dt>Группы</dt>
<dd>{catalog.source.superclasses.length} superclass</dd>
</div>
<div>
<dt>Лицензия</dt>
<dd>{catalog.source.license}</dd>
</div>
</dl>
{catalog.source.admissionStatus === "downloading"
&& catalog.source.archive
&& downloadProgress !== null ? (
<div className="dataset-download" aria-label="Загрузка GOOSE validation">
<span style={{ width: `${downloadProgress * 100}%` }} />
<p>
{formatBytes(catalog.source.archive.bytesTransferred)}
{" из "}
{formatBytes(catalog.source.archive.totalBytes)}
</p>
</div>
) : null}
<footer>
<p>
{catalog.source.admissionStatus === "frame-ready"
? "Native scan и разметка прошли admission. Можно открыть независимый ground truth."
: catalog.source.admissionStatus === "downloading"
? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment."
: catalog.storage.admitted
? "Следующий шаг: загрузить validation archive на worker и проверить hash/license."
: "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}
</p>
<div>
<Button
size="compact"
variant="secondary"
disabled={catalog.source.admissionStatus !== "frame-ready"}
title={
catalog.source.admissionStatus === "frame-ready"
? "Открыть фрагменты, сканы и результаты анализа"
: "Размеченные сканы ещё не импортированы"
}
onClick={() => setPreviewOpen((value) => !value)}
<div className="dataset-library__entries">
{catalog.sources.map((source) => {
const downloadProgress = source.archive
? source.archive.bytesTransferred / source.archive.totalBytes
: null;
const opened = source.sourceId === openSourceId;
return (
<article
className="dataset-entry"
data-source={source.sourceKind}
key={source.sourceId}
>
{previewOpen ? "Закрыть датасет" : "Открыть датасет"}
</Button>
</div>
</footer>
</article>
<div className="dataset-entry__identity">
<span className="dataset-entry__mark">
{source.sourceKind === "goose" ? "G" : "R"}
</span>
<div>
<span>
PUBLIC · OFF-ROAD · LABELED
{source.sourceKind === "rellis" ? " · RESEARCH" : ""}
</span>
<h3>{source.displayName}</h3>
<p>{sourceDescription(source)}</p>
</div>
</div>
<dl>
<div>
<dt>Состояние</dt>
<dd>{admissionLabel(source.admissionStatus)}</dd>
</div>
<div>
<dt>Сканер</dt>
<dd>{source.sensor}</dd>
</div>
<div>
<dt>Размеченных сканов</dt>
<dd>{source.annotatedScanCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Лицензия</dt>
<dd>{source.license}</dd>
</div>
</dl>
{source.admissionStatus === "downloading"
&& source.archive
&& downloadProgress !== null ? (
<div
className="dataset-download"
aria-label={`Загрузка ${source.displayName}`}
>
<span style={{ width: `${downloadProgress * 100}%` }} />
<p>
{formatBytes(source.archive.bytesTransferred)}
{" из "}
{formatBytes(source.archive.totalBytes)}
</p>
</div>
) : null}
<footer>
<p>{sourceFooter(source, catalog.storage.admitted)}</p>
<div>
<Button
size="compact"
variant="secondary"
disabled={!sourceReady(source)}
title={
sourceReady(source)
? "Открыть содержимое источника"
: "Совместимый preview ещё не опубликован"
}
onClick={() => setOpenSourceId(
opened ? null : source.sourceId,
)}
>
{opened ? "Закрыть датасет" : "Открыть датасет"}
</Button>
</div>
</footer>
</article>
);
})}
</div>
</GlassSurface>
{previewOpen && selectedRunId ? (
{openSource?.sourceKind === "goose" && selectedRunId ? (
<>
{gooseRuns.length > 1 ? (
<div className="dataset-result-select">
@@ -267,12 +315,14 @@ export function DatasetGatewayWorkspace() {
) : null}
<GooseDatasetReview runId={selectedRunId} />
</>
) : previewOpen ? (
) : openSource?.sourceKind === "goose" ? (
<section className="polygon-review-unavailable">
<StatusBadge tone="warning">Review недоступен</StatusBadge>
<h3>Датасет принят, но результаты анализа не опубликованы</h3>
<h3>GOOSE принят, но анализ не опубликован</h3>
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
</section>
) : openSource?.sourceKind === "rellis" ? (
<RellisDatasetReview />
) : null}
<details className="dataset-contract">
@@ -24,6 +24,7 @@ export interface LidarGroundPointCloudFrame {
disagreement: number[];
candidateDisagreement?: number[];
groundTruthGround?: number[];
evaluationMask?: number[];
};
}
@@ -87,13 +88,18 @@ function frameColors(
);
} else if (mode === "ground-truth") {
const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
setRgb(
colors,
offset,
groundTruth ? 0.76 : 0.29,
groundTruth ? 0.86 : 0.33,
groundTruth ? 0.52 : 0.36,
);
const evaluated = frame.masks.evaluationMask?.[index] !== 0;
if (!evaluated) {
setRgb(colors, offset, 0.86, 0.61, 0.24);
} else {
setRgb(
colors,
offset,
groundTruth ? 0.76 : 0.29,
groundTruth ? 0.86 : 0.33,
groundTruth ? 0.52 : 0.36,
);
}
} else if (mode === "current") {
setRgb(
colors,
@@ -0,0 +1,239 @@
import { useEffect, useMemo, useState } from "react";
import { StatusBadge } from "@nodedc/ui-react";
import {
fetchDatasetNativeScanPreview,
type DatasetNativeScanPreview,
} from "../core/lidar/datasetGateway";
import {
LidarGroundPointCloud,
type LidarGroundViewMode,
} from "./LidarGroundPointCloud";
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
["intensity", "Исходный скан"],
["semantic", "Классы RELLIS"],
["ground-truth", "Ground target"],
];
function formatPercent(value: number): string {
return new Intl.NumberFormat("ru-RU", {
style: "percent",
maximumFractionDigits: 1,
}).format(value);
}
function errorMessage(error: unknown): string {
return error instanceof Error && error.message.trim()
? error.message
: "Официальный пример RELLIS недоступен.";
}
function modeExplanation(mode: LidarGroundViewMode): string {
if (mode === "semantic") {
return "Цвета из закреплённой официальной ontology RELLIS-3D.";
}
if (mode === "ground-truth") {
return "Зелёный — ground, серый — non-ground, янтарный — ignore.";
}
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
}
export function RellisDatasetReview() {
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
const [error, setError] = useState<string | null>(null);
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
useEffect(() => {
const controller = new AbortController();
setError(null);
void fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
signal: controller.signal,
})
.then((value) => {
if (!controller.signal.aborted) setPreview(value);
})
.catch((loadError: unknown) => {
if (!controller.signal.aborted) setError(errorMessage(loadError));
});
return () => controller.abort();
}, []);
const scene = useMemo(() => {
if (!preview) return null;
const empty = Array.from({ length: preview.pointCount }, () => 0);
return {
pointCount: preview.pointCount,
pointsXyzM: preview.pointsXyzM,
intensity0To255: preview.remission0To255,
semanticRgb0To255: preview.semanticRgb0To255,
masks: {
currentGround: empty,
currentAssigned: preview.evaluationMask,
candidateGround: empty,
candidateAssigned: preview.evaluationMask,
disagreement: empty,
groundTruthGround: preview.groundTruthGround,
evaluationMask: preview.evaluationMask,
},
};
}, [preview]);
if (error) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="danger">Smoke preview недоступен</StatusBadge>
<h3>RELLIS-3D не открыт</h3>
<p>{error}</p>
</section>
);
}
if (!preview || !scene) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="accent">Проверяем пример</StatusBadge>
<h3>Открываем официальный RELLIS scan</h3>
<p>Исходные dataset bytes не переносятся в браузер.</p>
</section>
);
}
const evaluatedPoints = preview.evaluationMask.reduce(
(total, value) => total + value,
0,
);
const groundPoints = preview.groundTruthGround.reduce(
(total, value, index) => total + value * (preview.evaluationMask[index] ?? 0),
0,
);
const groundClasses = preview.classes.filter(
(item) => item.groundTarget === "ground",
);
const nonGroundClasses = preview.classes.filter(
(item) => item.groundTarget === "non-ground",
);
const ignoredClasses = preview.classes.filter(
(item) => item.groundTarget === "ignore",
);
return (
<section
className="dataset-sequence-review rellis-review"
aria-label="Просмотр официального примера RELLIS-3D"
>
<header className="dataset-sequence-review__truth">
<div>
<span className="section-eyebrow">RELLIS S0 · COMPATIBILITY SMOKE</span>
<h2>Официальный Ouster scan читается без конвертации формата</h2>
<p>
Frame {preview.frameId}: исходные XYZI и point-wise labels остались
выровнены. Это проверка reader, осей, цветов и ground-policy — ещё не
сравнение алгоритмов и не основание для навигации.
</p>
</div>
<dl>
<div>
<dt>Исходных точек</dt>
<dd>{preview.sourcePointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Browser preview</dt>
<dd>{preview.pointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Sensor frame</dt>
<dd>{preview.coordinateFrame?.frameId ?? "—"}</dd>
</div>
</dl>
</header>
<section className="polygon-review-player">
<header className="polygon-review-player__header">
<div>
<span className="section-eyebrow">OFFICIAL EXAMPLE · FRAME 000104</span>
<strong>Ouster OS1 64 · X forward · Y left · Z up</strong>
</div>
<div className="polygon-review-mode" aria-label="Режим RELLIS preview">
{modes.map(([viewMode, label]) => (
<button
type="button"
key={viewMode}
data-active={mode === viewMode ? "true" : undefined}
onClick={() => setMode(viewMode)}
>
{label}
</button>
))}
</div>
</header>
<div className="polygon-review-stage">
<LidarGroundPointCloud
frame={scene}
mode={mode}
coordinateFrame="sensor-fixed"
/>
<div className="polygon-review-stage__legend">
<strong>{modes.find(([viewMode]) => viewMode === mode)?.[1]}</strong>
<span>{modeExplanation(mode)}</span>
</div>
</div>
</section>
<div className="rellis-review__metrics">
<article>
<span>Оценимых точек preview</span>
<strong>{formatPercent(evaluatedPoints / preview.pointCount)}</strong>
<small>ignore не участвует в метриках</small>
</article>
<article>
<span>Ground среди оценимых</span>
<strong>
{formatPercent(evaluatedPoints ? groundPoints / evaluatedPoints : 0)}
</strong>
<small>версионная mapping policy v1</small>
</article>
<article>
<span>Классов в этом scan</span>
<strong>{preview.classes.length}</strong>
<small>из 20 классов ontology</small>
</article>
</div>
<section className="rellis-review__policy" aria-label="Ground target mapping">
<header>
<span className="section-eyebrow">GROUND TARGET POLICY V1</span>
<h3>Что именно будет оцениваться</h3>
<p>
Mapping принадлежит Mission Core и версионируется отдельно от
ontology датасета. Вода, лужи, void, sky и неопределённый object не
засчитываются ни как ground, ни как obstacle.
</p>
</header>
<div>
<article>
<span>Ground</span>
<strong>{groundClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
<article>
<span>Non-ground</span>
<strong>
{nonGroundClasses.map((item) => item.className).join(" · ") || "—"}
</strong>
</article>
<article>
<span>Ignore</span>
<strong>{ignoredClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
</div>
</section>
<p className="dataset-sequence-review__raw-note">
Лицензия RELLIS-3D — CC-BY-NC-SA-3.0: источник годится для независимой
исследовательской проверки, но коммерческое использование данных или
производных моделей требует отдельного лицензионного решения.
</p>
</section>
);
}
@@ -35,7 +35,7 @@ after(async () => {
function catalog(overrides = {}) {
return {
schema_version: "missioncore.dataset-gateway-catalog/v2",
schema_version: "missioncore.dataset-gateway-catalog/v3",
access: "read-only",
storage: {
configured: false,
@@ -49,19 +49,60 @@ function catalog(overrides = {}) {
},
sources: [{
source_id: "goose-3d/v2025-08-22",
source_kind: "goose",
display_name: "GOOSE 3D",
role: "primary-offroad-semantic-baseline",
license: "CC-BY-SA-4.0",
commercial_use: "allowed-with-share-alike",
format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution",
sensor: "VLS-128",
platforms: ["MuCAR-3", "ALICE", "Spot"],
annotations: ["semantic-point", "instance-point"],
superclasses: ["natural-ground", "obstacle"],
download: {
automatic: false,
reason: "operator-admitted-large-artifact-only",
smoke_example_mb: null,
primary_scan_archive_gb: 3.3,
label_archive_gb: null,
poses_archive_gb: null,
validation_archive_gb: 3.3,
},
statistics: {
fragment_count: 8,
annotated_scan_count: 961,
},
admission: {
status: "blocked-storage-policy",
archive: null,
frame: null,
},
}, {
source_id: "rellis-3d/v1.1",
source_kind: "rellis",
display_name: "RELLIS-3D",
role: "independent-offroad-cross-dataset-check",
license: "CC-BY-NC-SA-3.0",
commercial_use: "research-only-license-review-required",
format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution",
sensor: "Ouster OS1 64",
platforms: ["Clearpath Warthog"],
annotations: ["semantic-point"],
superclasses: ["ground", "obstacle", "ignore"],
download: {
automatic: false,
reason: "operator-admitted-large-artifact-only",
smoke_example_mb: 24,
primary_scan_archive_gb: 14,
label_archive_gb: 0.174,
poses_archive_gb: 0.174,
},
statistics: {
fragment_count: 5,
annotated_scan_count: 13_556,
},
admission: {
status: "blocked-storage-policy",
archive: null,
@@ -115,7 +156,8 @@ test("parses three distinct LiDAR representations and current-input boundary", (
);
assert.equal(parsed.representations[2].accumulation, true);
assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
assert.equal(parsed.source.frameSemantics, "one-lidar-revolution");
assert.equal(parsed.sources[0].frameSemantics, "one-lidar-revolution");
assert.equal(parsed.sources[1].sourceKind, "rellis");
});
test("rejects path exposure and upgraded K1 semantics", () => {
@@ -147,7 +189,7 @@ test("fetches the read-only gateway endpoint", async () => {
});
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
assert.equal(calls[0].init.method, "GET");
assert.equal(parsed.source.displayName, "GOOSE 3D");
assert.equal(parsed.sources[0].displayName, "GOOSE 3D");
});
test("decodes one bounded point-aligned native-scan preview", async () => {
@@ -182,10 +224,19 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
assert.equal(parsed.pointCount, 2);
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
let requestedUrl = null;
const fetched = await fetchDatasetNativeScanPreview({
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
});
assert.equal(fetched.frameId, "frame-1");
assert.deepEqual(fetched.evaluationMask, [1, 1]);
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=goose-3d%2Fv2025-08-22",
);
payload.semantic_rgb_0_to_255.pop();
assert.throws(
@@ -194,6 +245,92 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
);
});
test("decodes the RELLIS compatibility smoke with explicit ignore mask", async () => {
const payload = {
schema_version: "missioncore.dataset-native-scan-preview/v2",
source_id: "rellis-3d/v1.1",
frame_id: "000104",
representation: "native-scan",
sampling: "deterministic-even-index",
source_point_count: 3,
point_count: 3,
points_xyz_m: [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
remission_0_to_255: [0, 127, 255],
semantic_label_ids: [3, 4, 31],
semantic_rgb_0_to_255: [0, 102, 0, 0, 255, 0, 239, 255, 134],
ground_truth_ground: [1, 0, 0],
evaluation_mask: [1, 1, 0],
classes: [
{
label_id: 3,
class_name: "grass",
hex: "#006600",
ground_target: "ground",
source_point_count: 1,
},
{
label_id: 4,
class_name: "tree",
hex: "#00ff00",
ground_target: "non-ground",
source_point_count: 1,
},
{
label_id: 31,
class_name: "puddle",
hex: "#efff86",
ground_target: "ignore",
source_point_count: 1,
},
],
coordinate_frame: {
frame_id: "sensor/lidar/os1",
handedness: "right",
x: "forward",
y: "left",
z: "up",
transform_applied: false,
},
ground_policy: {
schema_version: "missioncore.rellis-ground-target-policy/v1",
ground: ["grass"],
non_ground: ["tree"],
ignore: ["puddle"],
},
source_evidence: {
repository_url: "https://github.com/unmannedlab/RELLIS-3D",
repository_commit: "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
label_config_sha256: "5".repeat(64),
point_sha256: "e".repeat(64),
label_sha256: "9".repeat(64),
license: "CC-BY-NC-SA-3.0",
},
safety: {
visualization_only: true,
compatibility_smoke_only: true,
navigation_or_safety_accepted: false,
},
};
const parsed = parseDatasetNativeScanPreview(payload);
assert.equal(parsed.sourceId, "rellis-3d/v1.1");
assert.deepEqual(parsed.evaluationMask, [1, 1, 0]);
assert.equal(parsed.classes[2].groundTarget, "ignore");
assert.equal(parsed.coordinateFrame.frameId, "sensor/lidar/os1");
let requestedUrl = null;
await fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
});
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=rellis-3d%2Fv1.1",
);
});
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
const payload = {
schema_version: "missioncore.dataset-ground-comparison-preview/v2",