feat: add RELLIS dataset compatibility smoke

This commit is contained in:
DCCONSTRUCTIONS 2026-07-25 20:45:30 +03:00
parent 13c35ff446
commit 054feec0d2
19 changed files with 1633 additions and 277 deletions

View File

@ -3,6 +3,57 @@ export type DatasetRepresentationId =
| "normalized-scan"
| "rolling-local-map";
export type DatasetSourceAdmissionStatus =
| "blocked-storage-policy"
| "ready-for-download"
| "ready-for-smoke"
| "downloading"
| "downloaded"
| "verifying"
| "verified"
| "frame-ready"
| "smoke-ready";
export interface DatasetSource {
sourceId: string;
sourceKind: "goose" | "rellis";
displayName: string;
role: string;
license: string;
commercialUse:
| "allowed-with-share-alike"
| "research-only-license-review-required";
format: string;
frameSemantics: "one-lidar-revolution";
sensor: string;
platforms: string[];
superclasses: string[];
smokeExampleMb: number | null;
primaryScanArchiveGb: number;
labelArchiveGb: number | null;
posesArchiveGb: number | null;
fragmentCount: number;
annotatedScanCount: number;
admissionStatus: DatasetSourceAdmissionStatus;
archive: {
bytesTransferred: number;
totalBytes: number;
sizeBytes: number | null;
sha256: string | null;
integrity: string;
vendorChecksumAvailable: false;
} | null;
frame: {
frameId: string;
pointCount: number;
semanticClassCount: number;
groundTruthGroundFraction: number;
previewPointCount: number;
previewSha256: string | null;
previewAvailable: true;
} | null;
}
export interface DatasetGatewayCatalog {
storage: {
configured: boolean;
@ -13,42 +64,7 @@ export interface DatasetGatewayCatalog {
requiredWindowsRoot: string;
requiredWslRoot: string;
};
source: {
sourceId: string;
displayName: string;
role: string;
license: string;
format: string;
frameSemantics: "one-lidar-revolution";
platforms: string[];
superclasses: string[];
validationArchiveGb: number;
admissionStatus:
| "blocked-storage-policy"
| "ready-for-download"
| "downloading"
| "downloaded"
| "verifying"
| "verified"
| "frame-ready";
archive: {
bytesTransferred: number;
totalBytes: number;
sizeBytes: number | null;
sha256: string | null;
integrity: string;
vendorChecksumAvailable: false;
} | null;
frame: {
frameId: string;
pointCount: number;
semanticClassCount: number;
groundTruthGroundFraction: number;
previewPointCount: number;
previewSha256: string;
previewAvailable: true;
} | null;
};
sources: DatasetSource[];
representations: Array<{
id: DatasetRepresentationId;
title: string;
@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
}
export interface DatasetNativeScanPreview {
sourceId: "goose-3d/v2025-08-22";
sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
frameId: string;
sourcePointCount: number;
pointCount: number;
@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
semanticLabelIds: number[];
semanticRgb0To255: number[];
groundTruthGround: number[];
evaluationMask: number[];
classes: Array<{
labelId: number;
className: string;
hex: string;
challengeCategoryId: number;
challengeCategoryName: string;
challengeCategoryId: number | null;
challengeCategoryName: string | null;
groundTarget: "ground" | "non-ground" | "ignore" | null;
sourcePointCount: number | null;
}>;
coordinateFrame: {
frameId: string;
x: "forward";
y: "left";
z: "up";
} | null;
compatibilitySmokeOnly: boolean;
}
export interface DatasetGroundMetrics {
@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
): DatasetGatewayCatalog {
const source = record(value, "Dataset Gateway");
if (
source.schema_version !== "missioncore.dataset-gateway-catalog/v2"
source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
|| source.access !== "read-only"
) {
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
@ -247,85 +273,158 @@ export function parseDatasetGatewayCatalog(
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
}
const sources = array(source.sources, "sources");
if (sources.length !== 1) {
throw new DatasetGatewayContractError("Ожидался один первичный dataset source");
if (sources.length !== 2) {
throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
}
const dataset = record(sources[0], "sources[0]");
const download = record(dataset.download, "source.download");
const admission = record(dataset.admission, "source.admission");
if (download.automatic !== false) {
throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически");
}
const admissionStatus = admission.status;
const parsedSources = sources.map((value, index): DatasetSource => {
const dataset = record(value, `sources[${index}]`);
const sourceKind = string(dataset.source_kind, "source_kind", true);
const sourceId = string(dataset.source_id, "source_id", true);
if (
(sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
&& (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
) {
throw new DatasetGatewayContractError("Dataset source identity неизвестна");
}
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
}
const download = record(dataset.download, `sources[${index}].download`);
const statistics = record(dataset.statistics, `sources[${index}].statistics`);
const admission = record(dataset.admission, `sources[${index}].admission`);
if (download.automatic !== false) {
throw new DatasetGatewayContractError(
"Большой dataset нельзя загружать автоматически",
);
}
const admissionStatus = admission.status;
if (
admissionStatus !== "ready-for-download"
&& admissionStatus !== "ready-for-smoke"
&& admissionStatus !== "blocked-storage-policy"
&& admissionStatus !== "downloading"
&& admissionStatus !== "downloaded"
&& admissionStatus !== "verifying"
&& admissionStatus !== "verified"
&& admissionStatus !== "frame-ready"
&& admissionStatus !== "smoke-ready"
) {
throw new DatasetGatewayContractError("source admission status неизвестен");
}
const archive = admission.archive === null
? null
: record(admission.archive, `sources[${index}].admission.archive`);
const archiveValue = archive
? {
bytesTransferred: number(
archive.bytes_transferred,
"archive.bytes_transferred",
),
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
sha256: nullableDigest(archive.sha256, "archive.sha256"),
integrity: string(archive.integrity, "archive.integrity", true),
vendorChecksumAvailable: (() => {
if (archive.vendor_checksum_available !== false) {
throw new DatasetGatewayContractError(
"archive.vendor_checksum_available: ожидался false",
);
}
return false as const;
})(),
}
: null;
const admittedFrame = admission.frame === null
? null
: record(admission.frame, `sources[${index}].admission.frame`);
const frameValue = admittedFrame
? {
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
pointCount: number(admittedFrame.point_count, "frame.point_count"),
semanticClassCount: number(
admittedFrame.semantic_class_count,
"frame.semantic_class_count",
),
groundTruthGroundFraction: number(
admittedFrame.ground_truth_ground_fraction,
"frame.ground_truth_ground_fraction",
),
previewPointCount: number(
admittedFrame.preview_point_count,
"frame.preview_point_count",
),
previewSha256: nullableDigest(
admittedFrame.preview_sha256,
"frame.preview_sha256",
),
previewAvailable: (() => {
if (admittedFrame.preview_available !== true) {
throw new DatasetGatewayContractError(
"frame.preview_available: ожидался true",
);
}
return true as const;
})(),
}
: null;
if (
sourceKind === "goose"
&& frameValue
&& frameValue.previewSha256 === null
) {
throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
}
const commercialUse = dataset.commercial_use;
if (
commercialUse !== "allowed-with-share-alike"
&& commercialUse !== "research-only-license-review-required"
) {
throw new DatasetGatewayContractError("Dataset license scope неизвестен");
}
return {
sourceId,
sourceKind,
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
commercialUse,
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
sensor: string(dataset.sensor, "sensor"),
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
smokeExampleMb: nullableNumber(
download.smoke_example_mb,
"smoke_example_mb",
),
primaryScanArchiveGb: number(
download.primary_scan_archive_gb,
"primary_scan_archive_gb",
),
labelArchiveGb: nullableNumber(
download.label_archive_gb,
"label_archive_gb",
),
posesArchiveGb: nullableNumber(
download.poses_archive_gb,
"poses_archive_gb",
),
fragmentCount: number(statistics.fragment_count, "fragment_count"),
annotatedScanCount: number(
statistics.annotated_scan_count,
"annotated_scan_count",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
};
});
if (
admissionStatus !== "ready-for-download"
&& admissionStatus !== "blocked-storage-policy"
&& admissionStatus !== "downloading"
&& admissionStatus !== "downloaded"
&& admissionStatus !== "verifying"
&& admissionStatus !== "verified"
&& admissionStatus !== "frame-ready"
parsedSources[0]?.sourceKind !== "goose"
|| parsedSources[1]?.sourceKind !== "rellis"
) {
throw new DatasetGatewayContractError("source admission status неизвестен");
throw new DatasetGatewayContractError("Dataset source order несовместим");
}
const archive = admission.archive === null
? null
: record(admission.archive, "source.admission.archive");
const archiveValue = archive
? {
bytesTransferred: number(
archive.bytes_transferred,
"archive.bytes_transferred",
),
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
sha256: nullableDigest(archive.sha256, "archive.sha256"),
integrity: string(archive.integrity, "archive.integrity", true),
vendorChecksumAvailable: (() => {
if (archive.vendor_checksum_available !== false) {
throw new DatasetGatewayContractError(
"archive.vendor_checksum_available: ожидался false",
);
}
return false as const;
})(),
}
: null;
const admittedFrame = admission.frame === null
? null
: record(admission.frame, "source.admission.frame");
const frameValue = admittedFrame
? {
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
pointCount: number(admittedFrame.point_count, "frame.point_count"),
semanticClassCount: number(
admittedFrame.semantic_class_count,
"frame.semantic_class_count",
),
groundTruthGroundFraction: number(
admittedFrame.ground_truth_ground_fraction,
"frame.ground_truth_ground_fraction",
),
previewPointCount: number(
admittedFrame.preview_point_count,
"frame.preview_point_count",
),
previewSha256: nullableDigest(
admittedFrame.preview_sha256,
"frame.preview_sha256",
) ?? (() => {
throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
})(),
previewAvailable: (() => {
if (admittedFrame.preview_available !== true) {
throw new DatasetGatewayContractError(
"frame.preview_available: ожидался true",
);
}
return true as const;
})(),
}
: null;
const representations = array(
source.representations,
"representations",
@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
) {
throw new DatasetGatewayContractError("Vendor-map boundary завышен");
}
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
}
return {
storage: {
configured: boolean(storage.configured, "storage.configured"),
@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
),
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
},
source: {
sourceId: string(dataset.source_id, "source_id", true),
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
validationArchiveGb: number(
download.validation_archive_gb,
"validation_archive_gb",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
},
sources: parsedSources,
representations,
pipeline,
currentInput: {
@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
value: unknown,
): DatasetNativeScanPreview {
const source = record(value, "Dataset preview");
const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
&& source.source_id === "goose-3d/v2025-08-22";
const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
&& source.source_id === "rellis-3d/v1.1";
if (
source.schema_version !== "missioncore.dataset-native-scan-preview/v1"
|| source.source_id !== "goose-3d/v2025-08-22"
(!isGoose && !isRellis)
|| source.representation !== "native-scan"
|| source.sampling !== "deterministic-even-index"
) {
@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
"ground_truth_ground",
1,
);
const evaluationMask = isRellis
? integers(source.evaluation_mask, "evaluation_mask", 1)
: Array.from({ length: pointCount }, () => 1);
if (
pointCount < 1
|| pointCount > 50_000
@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
|| semanticLabelIds.length !== pointCount
|| semanticRgb0To255.length !== pointCount * 3
|| groundTruthGround.length !== pointCount
|| evaluationMask.length !== pointCount
) {
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
}
const classes = array(source.classes, "classes").map((value, index) => {
const item = record(value, `classes[${index}]`);
const groundTarget = item.ground_target;
if (
groundTarget !== undefined
&& groundTarget !== "ground"
&& groundTarget !== "non-ground"
&& groundTarget !== "ignore"
) {
throw new DatasetGatewayContractError("class.ground_target неизвестен");
}
const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
groundTarget === undefined ? null : groundTarget;
return {
labelId: number(item.label_id, "class.label_id"),
className: string(item.class_name, "class.class_name", true),
hex: string(item.hex, "class.hex"),
challengeCategoryId: number(
item.challenge_category_id,
"class.challenge_category_id",
),
challengeCategoryName: string(
item.challenge_category_name,
"class.challenge_category_name",
true,
),
challengeCategoryId: item.challenge_category_id === undefined
? null
: number(item.challenge_category_id, "class.challenge_category_id"),
challengeCategoryName: item.challenge_category_name === undefined
? null
: string(
item.challenge_category_name,
"class.challenge_category_name",
true,
),
groundTarget: normalizedGroundTarget,
sourcePointCount: item.source_point_count === undefined
? null
: number(item.source_point_count, "class.source_point_count"),
};
});
const safety = record(source.safety, "safety");
if (
safety.visualization_only !== true
|| safety.navigation_or_safety_accepted !== false
|| (isRellis && safety.compatibility_smoke_only !== true)
) {
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
}
const coordinateFrame = isRellis
? (() => {
const frame = record(source.coordinate_frame, "coordinate_frame");
if (
frame.frame_id !== "sensor/lidar/os1"
|| frame.x !== "forward"
|| frame.y !== "left"
|| frame.z !== "up"
|| frame.handedness !== "right"
|| frame.transform_applied !== false
) {
throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
}
return {
frameId: "sensor/lidar/os1",
x: "forward" as const,
y: "left" as const,
z: "up" as const,
};
})()
: null;
return {
sourceId: "goose-3d/v2025-08-22",
sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
frameId: string(source.frame_id, "frame_id", true),
sourcePointCount,
pointCount,
@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
semanticLabelIds,
semanticRgb0To255,
groundTruthGround,
evaluationMask,
classes,
coordinateFrame,
compatibilitySmokeOnly: isRellis,
};
}
@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
}
export async function fetchDatasetNativeScanPreview(
options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {},
options: {
sourceId?: DatasetNativeScanPreview["sourceId"];
signal?: AbortSignal;
fetcher?: DatasetFetch;
} = {},
): Promise<DatasetNativeScanPreview> {
const fetcher = options.fetcher ?? fetch;
const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", {
const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
const response = await fetcher(
`/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
{
method: "GET",
headers: { Accept: "application/json" },
signal: options.signal,
});
},
);
return parseDatasetNativeScanPreview(await responseJson(response));
}

View File

@ -224,7 +224,9 @@
}
.dataset-fragment-comparison,
.dataset-fragment-risks {
.dataset-fragment-risks,
.rellis-review__metrics,
.rellis-review__policy > div {
grid-template-columns: 1fr;
}

View File

@ -1223,6 +1223,85 @@
line-height: 1.5;
}
.rellis-review__metrics {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__metrics article {
display: grid;
gap: 0.2rem;
border-radius: 0.85rem;
background: var(--station-panel);
padding: 0.78rem 0.85rem;
}
.rellis-review__metrics span,
.rellis-review__metrics small {
color: var(--nodedc-text-muted);
font-size: 0.55rem;
}
.rellis-review__metrics strong {
color: var(--nodedc-text-primary);
font-size: 1.05rem;
}
.rellis-review__policy {
display: grid;
gap: 0.7rem;
border-radius: 1rem;
background: var(--station-panel);
padding: 0.9rem 1rem;
}
.rellis-review__policy h3,
.rellis-review__policy p {
margin: 0;
}
.rellis-review__policy h3 {
margin-top: 0.28rem;
color: var(--nodedc-text-primary);
font-size: 0.95rem;
}
.rellis-review__policy p {
max-width: 52rem;
margin-top: 0.3rem;
color: var(--nodedc-text-muted);
font-size: 0.6rem;
line-height: 1.5;
}
.rellis-review__policy > div {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__policy article {
display: grid;
align-content: start;
gap: 0.35rem;
border-radius: 0.75rem;
background: rgb(255 255 255 / 0.03);
padding: 0.7rem 0.75rem;
}
.rellis-review__policy article span {
color: var(--nodedc-text-muted);
font-size: 0.54rem;
}
.rellis-review__policy article strong {
color: var(--nodedc-text-secondary);
font-size: 0.62rem;
font-weight: 560;
line-height: 1.5;
}
.dataset-sequence-timeline {
height: 2rem;
}
@ -2043,6 +2122,11 @@
font-size: 1rem;
}
.dataset-library__entries {
display: grid;
gap: 1rem;
}
.dataset-entry {
display: grid;
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);

View File

@ -8,12 +8,15 @@ import {
import {
fetchDatasetGatewayCatalog,
type DatasetGatewayCatalog,
type DatasetSource,
type DatasetSourceAdmissionStatus,
} from "../core/lidar/datasetGateway";
import {
fetchPolygonRunCatalog,
type PolygonRunCatalog,
} from "../core/polygon/runArchive";
import { GooseDatasetReview } from "./GooseDatasetReview";
import { RellisDatasetReview } from "./RellisDatasetReview";
const representationLabels: Record<string, string> = {
"native-scan": "Исходный скан",
@ -27,15 +30,17 @@ function errorMessage(error: unknown): string {
: "Каталог датасетов недоступен.";
}
function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) {
const labels: Record<typeof status, string> = {
function admissionLabel(status: DatasetSourceAdmissionStatus): string {
const labels: Record<DatasetSourceAdmissionStatus, string> = {
"blocked-storage-policy": "Worker не настроен",
"ready-for-download": "Готов к загрузке",
"ready-for-smoke": "Готов к smoke",
downloading: "Загружается",
downloaded: "Загружен",
verifying: "Проверяется",
verified: "Проверен",
"frame-ready": "Первый кадр готов",
"frame-ready": "Dataset готов",
"smoke-ready": "Smoke готов",
};
return labels[status];
}
@ -55,12 +60,42 @@ function isGooseQualificationRun(
&& item.scenarioGeneration.startsWith("goose-3d");
}
function sourceReady(source: DatasetSource): boolean {
return source.admissionStatus === "frame-ready"
|| source.admissionStatus === "smoke-ready";
}
function sourceDescription(source: DatasetSource): string {
if (source.sourceKind === "goose") {
return "Восемь фрагментов и 961 разреженный размеченный оборот VLS-128. Здесь уже опубликовано покадровое сравнение текущего ground baseline и Patchwork++.";
}
return "Независимый off-road источник с Ouster OS1 64. Сейчас открыт официальный frame 000104 для проверки reader, осей, ontology и ground-policy; полный benchmark ещё не выполнялся.";
}
function sourceFooter(source: DatasetSource, storageReady: boolean): string {
if (source.admissionStatus === "frame-ready") {
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
}
if (source.admissionStatus === "smoke-ready") {
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
}
if (source.admissionStatus === "downloading") {
return "Архив остаётся на D worker; после загрузки проверяются digest, лицензия и point-label alignment.";
}
if (!storageReady) {
return "Сначала нужен допущенный Dataset Root на диске D worker.";
}
return source.sourceKind === "rellis"
? "Следующий шаг: пропустить официальный 24 МБ example через smoke admission."
: "Следующий шаг: загрузить validation archive на worker и проверить hash/license.";
}
export function DatasetGatewayWorkspace() {
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [reloadGeneration, setReloadGeneration] = useState(0);
const [previewOpen, setPreviewOpen] = useState(false);
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
const [runError, setRunError] = useState<string | null>(null);
@ -85,7 +120,9 @@ export function DatasetGatewayWorkspace() {
}, [reloadGeneration]);
useEffect(() => {
if (catalog?.source.admissionStatus !== "downloading") return;
if (!catalog?.sources.some((source) => source.admissionStatus === "downloading")) {
return;
}
const timer = window.setTimeout(
() => setReloadGeneration((value) => value + 1),
5_000,
@ -113,10 +150,10 @@ export function DatasetGatewayWorkspace() {
return () => controller.abort();
}, []);
const downloadProgress = catalog?.source.archive
? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes
: null;
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
const openSource = catalog?.sources.find(
(source) => source.sourceId === openSourceId,
) ?? null;
return (
<div className="standard-workspace dataset-workspace">
@ -125,17 +162,17 @@ export function DatasetGatewayWorkspace() {
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
<h2>Независимая проверка алгоритмов</h2>
<p>
Датасет содержит известные размеченные данные. Мы показываем
конкретный фрагмент, результаты каждого алгоритма и ошибки
относительно разметки. Он не подмешивается в диагностику реального
сенсора и не заменяет closed-loop симуляцию.
Каждый источник остаётся отдельным: собственные sensor domain,
разметка, лицензия и результаты. Совпадение результата на GOOSE и
RELLIS снижает риск подгонки под один датасет, но не заменяет replay
реального сенсора и closed-loop симуляцию.
</p>
</div>
<ol aria-label="Как использовать датасет">
<li><span>01</span>Выбрать фрагмент</li>
<li><span>02</span>Посмотреть сканы</li>
<li><span>01</span>Выбрать источник</li>
<li><span>02</span>Проверить scan и labels</li>
<li><span>03</span>Сравнить алгоритмы</li>
<li><span>04</span>Проверить риски</li>
<li><span>04</span>Проверить переносимость</li>
</ol>
</section>
@ -171,80 +208,91 @@ export function DatasetGatewayWorkspace() {
</StatusBadge>
</header>
<article className="dataset-entry">
<div className="dataset-entry__identity">
<span className="dataset-entry__mark">G</span>
<div>
<span>PUBLIC · OFF-ROAD · LABELED</span>
<h3>{catalog.source.displayName}</h3>
<p>
Validation split: восемь фрагментов исходных записей и 961
разреженный размеченный оборот VLS-128. Это коллекция native
scans для perception, а не непрерывное видео или траектория
машины.
</p>
</div>
</div>
<dl>
<div>
<dt>Состояние</dt>
<dd>{admissionLabel(catalog.source.admissionStatus)}</dd>
</div>
<div>
<dt>Validation</dt>
<dd>{catalog.source.validationArchiveGb} ГБ</dd>
</div>
<div>
<dt>Группы</dt>
<dd>{catalog.source.superclasses.length} superclass</dd>
</div>
<div>
<dt>Лицензия</dt>
<dd>{catalog.source.license}</dd>
</div>
</dl>
{catalog.source.admissionStatus === "downloading"
&& catalog.source.archive
&& downloadProgress !== null ? (
<div className="dataset-download" aria-label="Загрузка GOOSE validation">
<span style={{ width: `${downloadProgress * 100}%` }} />
<p>
{formatBytes(catalog.source.archive.bytesTransferred)}
{" из "}
{formatBytes(catalog.source.archive.totalBytes)}
</p>
</div>
) : null}
<footer>
<p>
{catalog.source.admissionStatus === "frame-ready"
? "Native scan и разметка прошли admission. Можно открыть независимый ground truth."
: catalog.source.admissionStatus === "downloading"
? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment."
: catalog.storage.admitted
? "Следующий шаг: загрузить validation archive на worker и проверить hash/license."
: "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}
</p>
<div>
<Button
size="compact"
variant="secondary"
disabled={catalog.source.admissionStatus !== "frame-ready"}
title={
catalog.source.admissionStatus === "frame-ready"
? "Открыть фрагменты, сканы и результаты анализа"
: "Размеченные сканы ещё не импортированы"
}
onClick={() => setPreviewOpen((value) => !value)}
<div className="dataset-library__entries">
{catalog.sources.map((source) => {
const downloadProgress = source.archive
? source.archive.bytesTransferred / source.archive.totalBytes
: null;
const opened = source.sourceId === openSourceId;
return (
<article
className="dataset-entry"
data-source={source.sourceKind}
key={source.sourceId}
>
{previewOpen ? "Закрыть датасет" : "Открыть датасет"}
</Button>
</div>
</footer>
</article>
<div className="dataset-entry__identity">
<span className="dataset-entry__mark">
{source.sourceKind === "goose" ? "G" : "R"}
</span>
<div>
<span>
PUBLIC · OFF-ROAD · LABELED
{source.sourceKind === "rellis" ? " · RESEARCH" : ""}
</span>
<h3>{source.displayName}</h3>
<p>{sourceDescription(source)}</p>
</div>
</div>
<dl>
<div>
<dt>Состояние</dt>
<dd>{admissionLabel(source.admissionStatus)}</dd>
</div>
<div>
<dt>Сканер</dt>
<dd>{source.sensor}</dd>
</div>
<div>
<dt>Размеченных сканов</dt>
<dd>{source.annotatedScanCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Лицензия</dt>
<dd>{source.license}</dd>
</div>
</dl>
{source.admissionStatus === "downloading"
&& source.archive
&& downloadProgress !== null ? (
<div
className="dataset-download"
aria-label={`Загрузка ${source.displayName}`}
>
<span style={{ width: `${downloadProgress * 100}%` }} />
<p>
{formatBytes(source.archive.bytesTransferred)}
{" из "}
{formatBytes(source.archive.totalBytes)}
</p>
</div>
) : null}
<footer>
<p>{sourceFooter(source, catalog.storage.admitted)}</p>
<div>
<Button
size="compact"
variant="secondary"
disabled={!sourceReady(source)}
title={
sourceReady(source)
? "Открыть содержимое источника"
: "Совместимый preview ещё не опубликован"
}
onClick={() => setOpenSourceId(
opened ? null : source.sourceId,
)}
>
{opened ? "Закрыть датасет" : "Открыть датасет"}
</Button>
</div>
</footer>
</article>
);
})}
</div>
</GlassSurface>
{previewOpen && selectedRunId ? (
{openSource?.sourceKind === "goose" && selectedRunId ? (
<>
{gooseRuns.length > 1 ? (
<div className="dataset-result-select">
@ -267,12 +315,14 @@ export function DatasetGatewayWorkspace() {
) : null}
<GooseDatasetReview runId={selectedRunId} />
</>
) : previewOpen ? (
) : openSource?.sourceKind === "goose" ? (
<section className="polygon-review-unavailable">
<StatusBadge tone="warning">Review недоступен</StatusBadge>
<h3>Датасет принят, но результаты анализа не опубликованы</h3>
<h3>GOOSE принят, но анализ не опубликован</h3>
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
</section>
) : openSource?.sourceKind === "rellis" ? (
<RellisDatasetReview />
) : null}
<details className="dataset-contract">

View File

@ -24,6 +24,7 @@ export interface LidarGroundPointCloudFrame {
disagreement: number[];
candidateDisagreement?: number[];
groundTruthGround?: number[];
evaluationMask?: number[];
};
}
@ -87,13 +88,18 @@ function frameColors(
);
} else if (mode === "ground-truth") {
const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
setRgb(
colors,
offset,
groundTruth ? 0.76 : 0.29,
groundTruth ? 0.86 : 0.33,
groundTruth ? 0.52 : 0.36,
);
const evaluated = frame.masks.evaluationMask?.[index] !== 0;
if (!evaluated) {
setRgb(colors, offset, 0.86, 0.61, 0.24);
} else {
setRgb(
colors,
offset,
groundTruth ? 0.76 : 0.29,
groundTruth ? 0.86 : 0.33,
groundTruth ? 0.52 : 0.36,
);
}
} else if (mode === "current") {
setRgb(
colors,

View File

@ -0,0 +1,239 @@
import { useEffect, useMemo, useState } from "react";
import { StatusBadge } from "@nodedc/ui-react";
import {
fetchDatasetNativeScanPreview,
type DatasetNativeScanPreview,
} from "../core/lidar/datasetGateway";
import {
LidarGroundPointCloud,
type LidarGroundViewMode,
} from "./LidarGroundPointCloud";
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
["intensity", "Исходный скан"],
["semantic", "Классы RELLIS"],
["ground-truth", "Ground target"],
];
function formatPercent(value: number): string {
return new Intl.NumberFormat("ru-RU", {
style: "percent",
maximumFractionDigits: 1,
}).format(value);
}
function errorMessage(error: unknown): string {
return error instanceof Error && error.message.trim()
? error.message
: "Официальный пример RELLIS недоступен.";
}
function modeExplanation(mode: LidarGroundViewMode): string {
if (mode === "semantic") {
return "Цвета из закреплённой официальной ontology RELLIS-3D.";
}
if (mode === "ground-truth") {
return "Зелёный — ground, серый — non-ground, янтарный — ignore.";
}
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
}
export function RellisDatasetReview() {
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
const [error, setError] = useState<string | null>(null);
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
useEffect(() => {
const controller = new AbortController();
setError(null);
void fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
signal: controller.signal,
})
.then((value) => {
if (!controller.signal.aborted) setPreview(value);
})
.catch((loadError: unknown) => {
if (!controller.signal.aborted) setError(errorMessage(loadError));
});
return () => controller.abort();
}, []);
const scene = useMemo(() => {
if (!preview) return null;
const empty = Array.from({ length: preview.pointCount }, () => 0);
return {
pointCount: preview.pointCount,
pointsXyzM: preview.pointsXyzM,
intensity0To255: preview.remission0To255,
semanticRgb0To255: preview.semanticRgb0To255,
masks: {
currentGround: empty,
currentAssigned: preview.evaluationMask,
candidateGround: empty,
candidateAssigned: preview.evaluationMask,
disagreement: empty,
groundTruthGround: preview.groundTruthGround,
evaluationMask: preview.evaluationMask,
},
};
}, [preview]);
if (error) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="danger">Smoke preview недоступен</StatusBadge>
<h3>RELLIS-3D не открыт</h3>
<p>{error}</p>
</section>
);
}
if (!preview || !scene) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="accent">Проверяем пример</StatusBadge>
<h3>Открываем официальный RELLIS scan</h3>
<p>Исходные dataset bytes не переносятся в браузер.</p>
</section>
);
}
const evaluatedPoints = preview.evaluationMask.reduce(
(total, value) => total + value,
0,
);
const groundPoints = preview.groundTruthGround.reduce(
(total, value, index) => total + value * (preview.evaluationMask[index] ?? 0),
0,
);
const groundClasses = preview.classes.filter(
(item) => item.groundTarget === "ground",
);
const nonGroundClasses = preview.classes.filter(
(item) => item.groundTarget === "non-ground",
);
const ignoredClasses = preview.classes.filter(
(item) => item.groundTarget === "ignore",
);
return (
<section
className="dataset-sequence-review rellis-review"
aria-label="Просмотр официального примера RELLIS-3D"
>
<header className="dataset-sequence-review__truth">
<div>
<span className="section-eyebrow">RELLIS S0 · COMPATIBILITY SMOKE</span>
<h2>Официальный Ouster scan читается без конвертации формата</h2>
<p>
Frame {preview.frameId}: исходные XYZI и point-wise labels остались
выровнены. Это проверка reader, осей, цветов и ground-policy ещё не
сравнение алгоритмов и не основание для навигации.
</p>
</div>
<dl>
<div>
<dt>Исходных точек</dt>
<dd>{preview.sourcePointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Browser preview</dt>
<dd>{preview.pointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Sensor frame</dt>
<dd>{preview.coordinateFrame?.frameId ?? "—"}</dd>
</div>
</dl>
</header>
<section className="polygon-review-player">
<header className="polygon-review-player__header">
<div>
<span className="section-eyebrow">OFFICIAL EXAMPLE · FRAME 000104</span>
<strong>Ouster OS1 64 · X forward · Y left · Z up</strong>
</div>
<div className="polygon-review-mode" aria-label="Режим RELLIS preview">
{modes.map(([viewMode, label]) => (
<button
type="button"
key={viewMode}
data-active={mode === viewMode ? "true" : undefined}
onClick={() => setMode(viewMode)}
>
{label}
</button>
))}
</div>
</header>
<div className="polygon-review-stage">
<LidarGroundPointCloud
frame={scene}
mode={mode}
coordinateFrame="sensor-fixed"
/>
<div className="polygon-review-stage__legend">
<strong>{modes.find(([viewMode]) => viewMode === mode)?.[1]}</strong>
<span>{modeExplanation(mode)}</span>
</div>
</div>
</section>
<div className="rellis-review__metrics">
<article>
<span>Оценимых точек preview</span>
<strong>{formatPercent(evaluatedPoints / preview.pointCount)}</strong>
<small>ignore не участвует в метриках</small>
</article>
<article>
<span>Ground среди оценимых</span>
<strong>
{formatPercent(evaluatedPoints ? groundPoints / evaluatedPoints : 0)}
</strong>
<small>версионная mapping policy v1</small>
</article>
<article>
<span>Классов в этом scan</span>
<strong>{preview.classes.length}</strong>
<small>из 20 классов ontology</small>
</article>
</div>
<section className="rellis-review__policy" aria-label="Ground target mapping">
<header>
<span className="section-eyebrow">GROUND TARGET POLICY V1</span>
<h3>Что именно будет оцениваться</h3>
<p>
Mapping принадлежит Mission Core и версионируется отдельно от
ontology датасета. Вода, лужи, void, sky и неопределённый object не
засчитываются ни как ground, ни как obstacle.
</p>
</header>
<div>
<article>
<span>Ground</span>
<strong>{groundClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
<article>
<span>Non-ground</span>
<strong>
{nonGroundClasses.map((item) => item.className).join(" · ") || "—"}
</strong>
</article>
<article>
<span>Ignore</span>
<strong>{ignoredClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
</div>
</section>
<p className="dataset-sequence-review__raw-note">
Лицензия RELLIS-3D CC-BY-NC-SA-3.0: источник годится для независимой
исследовательской проверки, но коммерческое использование данных или
производных моделей требует отдельного лицензионного решения.
</p>
</section>
);
}

View File

@ -35,7 +35,7 @@ after(async () => {
function catalog(overrides = {}) {
return {
schema_version: "missioncore.dataset-gateway-catalog/v2",
schema_version: "missioncore.dataset-gateway-catalog/v3",
access: "read-only",
storage: {
configured: false,
@ -49,19 +49,60 @@ function catalog(overrides = {}) {
},
sources: [{
source_id: "goose-3d/v2025-08-22",
source_kind: "goose",
display_name: "GOOSE 3D",
role: "primary-offroad-semantic-baseline",
license: "CC-BY-SA-4.0",
commercial_use: "allowed-with-share-alike",
format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution",
sensor: "VLS-128",
platforms: ["MuCAR-3", "ALICE", "Spot"],
annotations: ["semantic-point", "instance-point"],
superclasses: ["natural-ground", "obstacle"],
download: {
automatic: false,
reason: "operator-admitted-large-artifact-only",
smoke_example_mb: null,
primary_scan_archive_gb: 3.3,
label_archive_gb: null,
poses_archive_gb: null,
validation_archive_gb: 3.3,
},
statistics: {
fragment_count: 8,
annotated_scan_count: 961,
},
admission: {
status: "blocked-storage-policy",
archive: null,
frame: null,
},
}, {
source_id: "rellis-3d/v1.1",
source_kind: "rellis",
display_name: "RELLIS-3D",
role: "independent-offroad-cross-dataset-check",
license: "CC-BY-NC-SA-3.0",
commercial_use: "research-only-license-review-required",
format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution",
sensor: "Ouster OS1 64",
platforms: ["Clearpath Warthog"],
annotations: ["semantic-point"],
superclasses: ["ground", "obstacle", "ignore"],
download: {
automatic: false,
reason: "operator-admitted-large-artifact-only",
smoke_example_mb: 24,
primary_scan_archive_gb: 14,
label_archive_gb: 0.174,
poses_archive_gb: 0.174,
},
statistics: {
fragment_count: 5,
annotated_scan_count: 13_556,
},
admission: {
status: "blocked-storage-policy",
archive: null,
@ -115,7 +156,8 @@ test("parses three distinct LiDAR representations and current-input boundary", (
);
assert.equal(parsed.representations[2].accumulation, true);
assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
assert.equal(parsed.source.frameSemantics, "one-lidar-revolution");
assert.equal(parsed.sources[0].frameSemantics, "one-lidar-revolution");
assert.equal(parsed.sources[1].sourceKind, "rellis");
});
test("rejects path exposure and upgraded K1 semantics", () => {
@ -147,7 +189,7 @@ test("fetches the read-only gateway endpoint", async () => {
});
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
assert.equal(calls[0].init.method, "GET");
assert.equal(parsed.source.displayName, "GOOSE 3D");
assert.equal(parsed.sources[0].displayName, "GOOSE 3D");
});
test("decodes one bounded point-aligned native-scan preview", async () => {
@ -182,10 +224,19 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
assert.equal(parsed.pointCount, 2);
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
let requestedUrl = null;
const fetched = await fetchDatasetNativeScanPreview({
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
});
assert.equal(fetched.frameId, "frame-1");
assert.deepEqual(fetched.evaluationMask, [1, 1]);
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=goose-3d%2Fv2025-08-22",
);
payload.semantic_rgb_0_to_255.pop();
assert.throws(
@ -194,6 +245,92 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
);
});
test("decodes the RELLIS compatibility smoke with explicit ignore mask", async () => {
const payload = {
schema_version: "missioncore.dataset-native-scan-preview/v2",
source_id: "rellis-3d/v1.1",
frame_id: "000104",
representation: "native-scan",
sampling: "deterministic-even-index",
source_point_count: 3,
point_count: 3,
points_xyz_m: [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
remission_0_to_255: [0, 127, 255],
semantic_label_ids: [3, 4, 31],
semantic_rgb_0_to_255: [0, 102, 0, 0, 255, 0, 239, 255, 134],
ground_truth_ground: [1, 0, 0],
evaluation_mask: [1, 1, 0],
classes: [
{
label_id: 3,
class_name: "grass",
hex: "#006600",
ground_target: "ground",
source_point_count: 1,
},
{
label_id: 4,
class_name: "tree",
hex: "#00ff00",
ground_target: "non-ground",
source_point_count: 1,
},
{
label_id: 31,
class_name: "puddle",
hex: "#efff86",
ground_target: "ignore",
source_point_count: 1,
},
],
coordinate_frame: {
frame_id: "sensor/lidar/os1",
handedness: "right",
x: "forward",
y: "left",
z: "up",
transform_applied: false,
},
ground_policy: {
schema_version: "missioncore.rellis-ground-target-policy/v1",
ground: ["grass"],
non_ground: ["tree"],
ignore: ["puddle"],
},
source_evidence: {
repository_url: "https://github.com/unmannedlab/RELLIS-3D",
repository_commit: "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
label_config_sha256: "5".repeat(64),
point_sha256: "e".repeat(64),
label_sha256: "9".repeat(64),
license: "CC-BY-NC-SA-3.0",
},
safety: {
visualization_only: true,
compatibility_smoke_only: true,
navigation_or_safety_accepted: false,
},
};
const parsed = parseDatasetNativeScanPreview(payload);
assert.equal(parsed.sourceId, "rellis-3d/v1.1");
assert.deepEqual(parsed.evaluationMask, [1, 1, 0]);
assert.equal(parsed.classes[2].groundTarget, "ignore");
assert.equal(parsed.coordinateFrame.frameId, "sensor/lidar/os1");
let requestedUrl = null;
await fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
});
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=rellis-3d%2Fv1.1",
);
});
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
const payload = {
schema_version: "missioncore.dataset-ground-comparison-preview/v2",

View File

@ -203,6 +203,14 @@ qualification bootstrap, never the product data plane. A shared
operator-facing deployment still requires authentication/RBAC, service
supervision and an accepted routed worker transport.
The Dataset Gateway is now multi-source. RELLIS S0 pins official repository
frame `000104`, verifies `131,072` Ouster XYZI points against aligned labels and
publishes a bounded semantic/ground-target preview beside GOOSE in the same
`Полигон → Датасеты` workflow. This closes only reader, axes, ontology,
licensing and versioned ignore-policy compatibility. Full RELLIS Ouster scans,
labels and poses remain D-only admission work before the cross-dataset
Current/Patchwork++ decision.
Mission Core remains a distributed web product. React is served by a long-lived
Gateway/API and renders canonical state through browser-native components.
Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the

View File

@ -878,6 +878,11 @@ workflow:
- `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is
explicitly grouped into eight independent fragments; Play is bounded to one
fragment and described as accelerated review of sparse annotated scans.
- The catalog is multi-source. RELLIS-3D v1.1 appears as an independent Ouster
domain with its own license and admission state. Its current `smoke-ready`
view shows official frame `000104`, semantic colors, FLU axes and the
versioned `ground / non-ground / ignore` mapping; it does not show invented
algorithm metrics.
- The dataset viewer exposes source frame number, nanosecond timestamp gaps,
ground truth and Current/Patchwork++ overlays.
- The fixed sensor-centric coordinate frame and operator camera are preserved
@ -894,6 +899,12 @@ The accepted `3.3 GB` annotated validation ZIP is not a continuous recording.
Continuous ego-motion requires a separately admitted raw ROS bag and
localization source.
RELLIS S0 uses only a bounded derivative of the pinned official example for
browser inspection. The full `14 GB` Ouster SemanticKITTI scans, labels and
poses remain a separate D-only admission gate. Its `CC-BY-NC-SA-3.0` license
means cross-dataset research evidence cannot silently become an unrestricted
commercial runtime or training dependency.
The backend reads the configured repository from
`MISSIONCORE_POLYGON_RUNS_ROOT` using
`QualificationRunStore(read_only=True)`. It exposes only:

View File

@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
domain adaptation after a public baseline proves that the pipeline and metric
harness work.
### L2.5 — Dataset Gateway — first-frame A/B complete
### L2.5 — Dataset Gateway — GOOSE qualification and RELLIS S0 complete
- [x] Define separate `native-scan`, `normalized-scan` and
`rolling-local-map` representations.
@ -317,12 +317,21 @@ harness work.
through an immutable R1 Polygon replay-shadow run.
- [x] Add named range/FOV/density/noise/dropout degradation profiles without
overwriting the native frame.
- [x] Upgrade the gateway to two independently identified sources instead of
treating GOOSE as a universal dataset contract.
- [x] Pin and verify the official RELLIS-3D Ouster example, preserve its
`131,072` point/label alignment and publish a bounded semantic viewer.
- [x] Version the RELLIS `ground / non-ground / ignore` mapping and expose
licensing as research-only evidence.
- [ ] Admit the full RELLIS Ouster SemanticKITTI scans, labels and poses on
worker D, then run the same Current/Patchwork++ harness.
The architectural contract and run sequence are fixed in
`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its
published 3D format is one LiDAR revolution with point-wise semantic and
instance labels in off-road environments. RELLIS-3D remains the second-source
cross-check after the GOOSE harness is stable.
instance labels in off-road environments. The RELLIS-3D compatibility smoke is
now complete against official frame `000104`; the full second-source algorithm
cross-check remains the next gate.
### L3 — LiDAR-native 3D detection
@ -394,7 +403,10 @@ The near-term value is not a prettier point cloud:
The GOOSE validation-split and deterministic degradation gates are complete:
pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises
natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95
across 961 frames. The decision remains shadow-only. The highest-value
immediate work is now a second-source cross-check plus LiDAR-native detection
on the same gateway. Nvblox and alternative SLAM remain later because their
timing, pose and scan-geometry gates are not yet satisfied.
across 961 frames. RELLIS S0 proves that the shared reader, axes, ontology and
ground-target mapping work on a different Ouster off-road domain; it does not
yet score an algorithm. The decision remains shadow-only. The highest-value
immediate work is full RELLIS ground qualification, followed by the real
sensor input contract and LiDAR-native detection on the same gateway. Nvblox
and alternative SLAM remain later because their timing, pose and scan-geometry
gates are not yet satisfied.

View File

@ -19,12 +19,13 @@ datasets and future real onboard sensors.
Implemented now:
- `missioncore.dataset-gateway-catalog/v2`, exposed read-only at
- multi-source `missioncore.dataset-gateway-catalog/v3`, exposed read-only at
`GET /api/v1/lidar/dataset-gateway`;
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a
static React status;
- a bounded `missioncore.dataset-native-scan-preview/v1`, exposed read-only at
`GET /api/v1/lidar/dataset-gateway/preview`;
- bounded `missioncore.dataset-native-scan-preview/v1` (GOOSE) and `/v2`
(RELLIS smoke) artifacts, selected by source at
`GET /api/v1/lidar/dataset-gateway/preview?source_id=...`;
- explicit `native-scan`, `normalized-scan` and `rolling-local-map`
representations;
- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI
@ -39,6 +40,10 @@ Implemented now:
`Открыть` action;
- one admitted GOOSE validation frame with source colors, normalized remission
and an independent ground-truth view;
- a pinned RELLIS-3D v1.1 compatibility smoke using the official Ouster example
frame `000104`, its native `131,072` XYZI points, point-aligned labels,
official ontology colors, explicit FLU sensor axes and a versioned
`ground / non-ground / ignore` evaluation policy;
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
@ -54,6 +59,8 @@ Not implemented:
- no implicit coordinate conversion;
- no fake ring/timestamp reconstruction for K1 MQTT evidence;
- no model training or production promotion;
- no full RELLIS Ouster archive, pose set, algorithm comparison or ROS bag
admission yet;
- no rolling-map implementation yet.
## Product surface boundary
@ -73,10 +80,11 @@ The gateway is not part of device quality diagnostics.
backend evidence contract. They are not a second operator navigation item
and are not presented as vehicle motion.
Before real dataset bytes exist, the catalog explains the blocked storage gate
and keeps `Открыть датасет` disabled. It becomes available
only after the worker manifest says `frame-ready` and the bounded preview passes
its own point-alignment and safety checks.
Before compatible source evidence exists, the catalog explains the blocked
storage gate and keeps `Открыть датасет` disabled. GOOSE becomes available only
after the worker manifest says `frame-ready`; RELLIS becomes available at
`smoke-ready` only for the pinned official example. Both bounded previews must
pass their own identity, point-alignment and safety checks.
## Why public recordings look different
@ -191,8 +199,53 @@ time; TTL/dynamic filtering prevents stale ghosts.
- [x] Patchwork++ one-frame accuracy measured against ground truth.
- [x] Current/Patchwork++ validation-split gate qualified.
- [x] Sensor-degradation matrix qualified.
- [x] Dataset catalog accepts more than one independently identified source.
- [x] Official RELLIS Ouster example passes the shared SemanticKITTI reader.
- [x] RELLIS axes, ontology colors and versioned ground-target/ignore mapping
are visible in the same Dataset workflow.
- [ ] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
worker D.
- [ ] Current/Patchwork++ RELLIS comparison qualified without obstacle loss.
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
## RELLIS S0 compatibility evidence
RELLIS is the independent second-source check, not extra decoration in the
catalog. GOOSE and RELLIS share the ingress format and viewer, but retain
separate source identity, sensor domain, ontology, licensing and results.
The pinned smoke input is the official repository example at commit
`c17a118fcaed1559f03cc32cc3a91dedc557f8b8`:
- point file: `utils/example/000104.bin`;
- label file: `utils/example/000104.label`;
- source points: `131,072`;
- point SHA-256:
`ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af`;
- label SHA-256:
`9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e`;
- official ontology config SHA-256:
`573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad`;
- bounded browser preview: `20,000` deterministic even-index points;
- coordinate frame: right-handed, `x` forward, `y` left, `z` up; no vehicle
transform, pose registration or deskew is claimed;
- license: `CC-BY-NC-SA-3.0`; this is a research qualification source, not an
unrestricted commercial runtime dependency.
Mission Core ground policy v1 treats `dirt`, `grass`, `asphalt`, `concrete` and
`mud` as ground; clear structures, vegetation, people, vehicles and rubble as
non-ground; and `void`, `water`, `sky`, generic `object` and `puddle` as
`ignore`. The ignore set is excluded from future metrics instead of being
silently counted as either free ground or obstacle. This mapping is a Mission
Core evaluation decision, not an upstream RELLIS claim.
The smoke closes reader, alignment, axes, palette and mapping compatibility
only. The next gate admits the `14 GB` Ouster SemanticKITTI scans plus the
`174 MB` labels and `174 MB` poses to D-only worker storage. Full ROS bags are
not needed for the ground cross-check. Patchwork++ may be compared only after a
physical Ouster mounting-height profile is evidenced; no height is inferred
from the example cloud.
## First GOOSE admission evidence
- worker root: canonical D-only root, path never exposed by the HTTP API;

View File

@ -39,6 +39,12 @@ first admitted source because it publishes off-road point-wise semantic and
instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its
annotated point-cloud file represents one LiDAR revolution.
The gateway contract is multi-source, not GOOSE-shaped. RELLIS-3D v1.1 is the
second source: it uses the same lossless SemanticKITTI ingress but keeps a
separate Ouster OS1 domain, ontology, admission state, ground-target mapping,
license and result set. Shared format never implies shared calibration,
mounting height or commercial-use rights.
The gateway:
- preserves the native GOOSE frame before adaptation;
@ -48,6 +54,10 @@ The gateway:
- refuses automatic downloads of large archives;
- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror;
- never promotes K1 `lio_pcl` to `native-scan`.
- excludes explicitly ambiguous RELLIS classes through a versioned `ignore`
mask instead of silently scoring them as ground or obstacle;
- exposes the RELLIS `CC-BY-NC-SA-3.0` restriction as research evidence and
never treats it as an unrestricted production dependency.
Large-artifact execution remains worker-local. The browser receives only a
path-free admission manifest and a bounded visualization preview. The canonical
@ -82,6 +92,8 @@ unavailable rather than inventing timestamps.
- Until bytes are installed, the UI exposes an explicit empty state. After a
`frame-ready` worker admission, the same surface becomes an interactive
source/remission/ground-truth viewer without changing the representation.
- A bounded `smoke-ready` source may prove reader, axes, ontology and mapping
compatibility without claiming full-dataset or algorithm qualification.
- Dataset and device inputs can share downstream algorithms only after their
normalized contracts match.
- Sensor adaptation may change range, FOV, point density, noise and dropout for
@ -115,6 +127,7 @@ unavailable rather than inventing timestamps.
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
- [RELLIS-3D official repository](https://github.com/unmannedlab/RELLIS-3D)
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)

View File

@ -38,6 +38,15 @@ from k1link.datasets.goose_qualification import (
GroundAcceptancePolicy,
qualify_goose_ground,
)
from k1link.datasets.rellis_smoke import (
RELLIS_CLASSES,
RELLIS_GROUND_POLICY_SCHEMA,
RELLIS_PREVIEW_SCHEMA,
RELLIS_SOURCE_ID,
RellisSmokeError,
build_rellis_official_smoke_preview,
rellis_native_scan_preview,
)
__all__ = [
"DATASET_GATEWAY_CATALOG_SCHEMA",
@ -54,12 +63,18 @@ __all__ = [
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
"RELLIS_CLASSES",
"RELLIS_GROUND_POLICY_SCHEMA",
"RELLIS_PREVIEW_SCHEMA",
"RELLIS_SOURCE_ID",
"RellisSmokeError",
"GoosePatchworkProfile",
"GroundAcceptancePolicy",
"DegradationProfile",
"DEFAULT_DEGRADATIONS",
"benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground",
"build_rellis_official_smoke_preview",
"configured_dataset_admission_manifest",
"configured_dataset_ground_preview",
"configured_dataset_preview",
@ -69,5 +84,6 @@ __all__ = [
"read_dataset_ground_preview",
"read_dataset_native_scan_preview",
"read_semantic_kitti_frame",
"rellis_native_scan_preview",
"qualify_goose_ground",
]

View File

@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
)
from k1link.datasets.goose_qualification import qualify_goose_ground
from k1link.datasets.goose_review import build_goose_ground_review_pack
from k1link.datasets.rellis_smoke import (
RellisSmokeError,
build_rellis_official_smoke_preview,
)
app = typer.Typer(
add_completion=False,
@ -158,5 +162,50 @@ def build_goose_ground_review_command(
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
@app.command("build-rellis-smoke-preview")
def build_rellis_smoke_preview_command(
points: Annotated[
Path,
typer.Option("--points", exists=True, dir_okay=False, resolve_path=True),
],
labels: Annotated[
Path,
typer.Option("--labels", exists=True, dir_okay=False, resolve_path=True),
],
out: Annotated[
Path,
typer.Option("--out", dir_okay=False, resolve_path=True),
],
preview_points: Annotated[
int,
typer.Option("--preview-points", min=1, max=50_000),
] = 20_000,
) -> None:
"""Verify the pinned official RELLIS example and publish a bounded preview."""
try:
preview = build_rellis_official_smoke_preview(
points,
labels,
out,
preview_points=preview_points,
)
except RellisSmokeError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(
json.dumps(
{
"source_id": preview["source_id"],
"frame_id": preview["frame_id"],
"source_point_count": preview["source_point_count"],
"preview_point_count": preview["point_count"],
},
ensure_ascii=False,
sort_keys=True,
)
)
if __name__ == "__main__":
app()

View File

@ -18,7 +18,7 @@ import numpy.typing as npt
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v3"
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
@ -177,6 +177,7 @@ def _is_worker_d_storage(root: Path | None) -> bool:
def dataset_gateway_catalog(
dataset_root: Path | None = None,
admission_manifest_path: Path | None = None,
rellis_preview_path: Path | None = None,
) -> dict[str, object]:
"""Return the path-free, read-only ingress plan and current admission state."""
@ -196,6 +197,14 @@ def dataset_gateway_catalog(
locally_admitted = _is_worker_d_storage(root)
worker_admitted = bool(admission and admission["storage"]["admitted"])
storage_admitted = locally_admitted or worker_admitted
rellis_preview: dict[str, Any] | None = None
if rellis_preview_path is not None and rellis_preview_path.is_file():
try:
candidate = read_dataset_native_scan_preview(rellis_preview_path)
if candidate.get("source_id") == "rellis-3d/v1.1":
rellis_preview = candidate
except DatasetAdmissionError:
pass
source_status = (
str(admission["status"])
if admission is not None
@ -212,6 +221,20 @@ def dataset_gateway_catalog(
if storage_admitted
else "configure-dataset-root-on-worker-d"
)
rellis_status = (
"smoke-ready"
if rellis_preview is not None
else "ready-for-smoke"
if storage_admitted
else "blocked-storage-policy"
)
catalog_next_action = (
"admit-rellis-ouster-semantickitti-to-worker-d"
if rellis_preview is not None and source_status == "frame-ready"
else "build-rellis-official-example-smoke"
if source_status == "frame-ready"
else next_action
)
return {
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
"access": "read-only",
@ -234,11 +257,14 @@ def dataset_gateway_catalog(
"sources": [
{
"source_id": "goose-3d/v2025-08-22",
"source_kind": "goose",
"display_name": "GOOSE 3D",
"role": "primary-offroad-semantic-baseline",
"license": "CC-BY-SA-4.0",
"commercial_use": "allowed-with-share-alike",
"format": "semantickitti-xyzi-label",
"frame_semantics": "one-lidar-revolution",
"sensor": "VLS-128",
"platforms": ["MuCAR-3", "ALICE", "Spot"],
"annotations": ["semantic-point", "instance-point"],
"superclasses": [
@ -255,10 +281,18 @@ def dataset_gateway_catalog(
"download": {
"automatic": False,
"reason": "operator-admitted-large-artifact-only",
"smoke_example_mb": None,
"primary_scan_archive_gb": 3.3,
"label_archive_gb": None,
"poses_archive_gb": None,
"training_archive_gb": 27.0,
"validation_archive_gb": 3.3,
"test_archive_gb": 3.3,
},
"statistics": {
"fragment_count": 8,
"annotated_scan_count": 961,
},
"admission": {
"status": source_status,
"native_scan": "ready-after-download",
@ -267,7 +301,72 @@ def dataset_gateway_catalog(
"archive": admission["archive"] if admission is not None else None,
"frame": admission["frame"] if admission is not None else None,
},
}
},
{
"source_id": "rellis-3d/v1.1",
"source_kind": "rellis",
"display_name": "RELLIS-3D",
"role": "independent-offroad-cross-dataset-check",
"license": "CC-BY-NC-SA-3.0",
"commercial_use": "research-only-license-review-required",
"format": "semantickitti-xyzi-label",
"frame_semantics": "one-lidar-revolution",
"sensor": "Ouster OS1 64",
"platforms": ["Clearpath Warthog"],
"annotations": ["semantic-point"],
"superclasses": [
"ground",
"vegetation",
"structure",
"obstacle",
"vehicle",
"human",
"water",
"ignore",
],
"download": {
"automatic": False,
"reason": "operator-admitted-large-artifact-only",
"smoke_example_mb": 24.0,
"primary_scan_archive_gb": 14.0,
"label_archive_gb": 0.174,
"poses_archive_gb": 0.174,
"training_archive_gb": None,
"validation_archive_gb": None,
"test_archive_gb": None,
},
"statistics": {
"fragment_count": 5,
"annotated_scan_count": 13_556,
},
"admission": {
"status": rellis_status,
"native_scan": (
"official-example-compatible"
if rellis_preview is not None
else "requires-official-example-smoke"
),
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
"rolling_local_map": "requires-poses-timing-and-map-policy",
"archive": None,
"frame": (
{
"frame_id": rellis_preview["frame_id"],
"point_count": rellis_preview["source_point_count"],
"semantic_class_count": len(rellis_preview["classes"]),
"ground_truth_ground_fraction": (
sum(rellis_preview["ground_truth_ground"])
/ rellis_preview["point_count"]
),
"preview_point_count": rellis_preview["point_count"],
"preview_sha256": None,
"preview_available": True,
}
if rellis_preview is not None
else None
),
},
},
],
"representations": [
{
@ -330,7 +429,7 @@ def dataset_gateway_catalog(
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
}
],
"next_action": next_action,
"next_action": catalog_next_action,
}
@ -417,9 +516,13 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
identity = (document.get("schema_version"), document.get("source_id"))
if (
document.get("schema_version") != DATASET_PREVIEW_SCHEMA
or document.get("source_id") != "goose-3d/v2025-08-22"
identity
not in {
(DATASET_PREVIEW_SCHEMA, "goose-3d/v2025-08-22"),
("missioncore.dataset-native-scan-preview/v2", "rellis-3d/v1.1"),
}
or document.get("representation") != "native-scan"
or document.get("sampling") != "deterministic-even-index"
):
@ -433,6 +536,11 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
semantic_ids = document.get("semantic_label_ids")
semantic_rgb = document.get("semantic_rgb_0_to_255")
ground = document.get("ground_truth_ground")
evaluated = (
document.get("evaluation_mask")
if identity[1] == "rellis-3d/v1.1"
else [1] * point_count
)
if (
not isinstance(points, list)
or len(points) != point_count
@ -444,6 +552,8 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
or len(semantic_rgb) != point_count * 3
or not isinstance(ground, list)
or len(ground) != point_count
or not isinstance(evaluated, list)
or len(evaluated) != point_count
):
raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
for point in points:
@ -463,6 +573,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
(semantic_ids, 65_535, "semantic label"),
(semantic_rgb, 255, "semantic color"),
(ground, 1, "ground mask"),
(evaluated, 1, "evaluation mask"),
):
if any(
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
@ -473,16 +584,81 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
if not isinstance(classes, list) or len(classes) > 64:
raise DatasetAdmissionError("dataset preview class catalog is incompatible")
safety = _object(document.get("safety"), "safety")
if safety != {
expected_safety = {
"visualization_only": True,
"navigation_or_safety_accepted": False,
}:
}
if identity[1] == "rellis-3d/v1.1":
expected_safety["compatibility_smoke_only"] = True
if safety != expected_safety:
raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
raise DatasetAdmissionError("dataset preview frame id is incompatible")
if identity[1] == "rellis-3d/v1.1":
_validate_rellis_preview_metadata(document, classes)
return document
def _validate_rellis_preview_metadata(
document: dict[str, Any],
classes: list[Any],
) -> None:
coordinate_frame = _object(document.get("coordinate_frame"), "coordinate_frame")
if coordinate_frame != {
"frame_id": "sensor/lidar/os1",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"transform_applied": False,
}:
raise DatasetAdmissionError("RELLIS coordinate frame is incompatible")
policy = _object(document.get("ground_policy"), "ground_policy")
if (
policy.get("schema_version") != "missioncore.rellis-ground-target-policy/v1"
or set(policy) != {"schema_version", "ground", "non_ground", "ignore"}
or any(
not isinstance(policy.get(key), list)
or any(not isinstance(value, str) or not value for value in policy[key])
for key in ("ground", "non_ground", "ignore")
)
):
raise DatasetAdmissionError("RELLIS ground target policy is incompatible")
for item in classes:
value = _object(item, "class")
if (
value.get("ground_target") not in {"ground", "non-ground", "ignore"}
or not isinstance(value.get("label_id"), int)
or not isinstance(value.get("class_name"), str)
or not isinstance(value.get("hex"), str)
or not isinstance(value.get("source_point_count"), int)
):
raise DatasetAdmissionError("RELLIS class catalog is incompatible")
evidence = _object(document.get("source_evidence"), "source_evidence")
if (
set(evidence)
!= {
"repository_url",
"repository_commit",
"label_config_sha256",
"point_sha256",
"label_sha256",
"license",
}
or evidence.get("repository_url") != "https://github.com/unmannedlab/RELLIS-3D"
or evidence.get("repository_commit")
!= "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
or evidence.get("label_config_sha256")
!= "573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
or evidence.get("point_sha256")
!= "ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
or evidence.get("label_sha256")
!= "9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
or evidence.get("license") != "CC-BY-NC-SA-3.0"
):
raise DatasetAdmissionError("RELLIS source evidence is incompatible")
def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
if (

View File

@ -0,0 +1,275 @@
"""Pinned RELLIS-3D compatibility smoke for the shared Dataset Gateway.
The smoke check deliberately stops before algorithm qualification. It proves
that Mission Core can read one official Ouster OS1 SemanticKITTI frame, preserve
point/label alignment, apply an explicit ground-evaluation policy and publish a
bounded visualization artifact. Full RELLIS archives and ROS bags remain
worker-D-only inputs.
"""
from __future__ import annotations
import hashlib
import json
import os
import tempfile
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final, Literal
import numpy as np
from k1link.datasets.gateway import DatasetFrameError, DatasetPointFrame, read_semantic_kitti_frame
RELLIS_SOURCE_ID: Final = "rellis-3d/v1.1"
RELLIS_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v2"
RELLIS_GROUND_POLICY_SCHEMA: Final = "missioncore.rellis-ground-target-policy/v1"
RELLIS_REPOSITORY_URL: Final = "https://github.com/unmannedlab/RELLIS-3D"
RELLIS_REPOSITORY_COMMIT: Final = "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
RELLIS_LABEL_CONFIG_SHA256: Final = (
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
)
RELLIS_EXAMPLE_POINTS_SHA256: Final = (
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
)
RELLIS_EXAMPLE_LABELS_SHA256: Final = (
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
)
RELLIS_EXAMPLE_FRAME_ID: Final = "000104"
RELLIS_LICENSE: Final = "CC-BY-NC-SA-3.0"
MAX_RELLIS_PREVIEW_POINTS: Final = 50_000
GroundTarget = Literal["ground", "non-ground", "ignore"]
@dataclass(frozen=True)
class RellisClass:
label_id: int
class_name: str
bgr: tuple[int, int, int]
ground_target: GroundTarget
@property
def rgb(self) -> tuple[int, int, int]:
blue, green, red = self.bgr
return red, green, blue
@property
def hex(self) -> str:
red, green, blue = self.rgb
return f"#{red:02x}{green:02x}{blue:02x}"
def to_preview_dict(self) -> dict[str, object]:
return {
"label_id": self.label_id,
"class_name": self.class_name,
"hex": self.hex,
"ground_target": self.ground_target,
}
# IDs, names and BGR colors are pinned to the official repository config at
# RELLIS_REPOSITORY_COMMIT. The final column is Mission Core's versioned,
# algorithm-scoped ground evaluation policy; it is not an upstream claim.
RELLIS_CLASSES: Final[tuple[RellisClass, ...]] = (
RellisClass(0, "void", (0, 0, 0), "ignore"),
RellisClass(1, "dirt", (108, 64, 20), "ground"),
RellisClass(3, "grass", (0, 102, 0), "ground"),
RellisClass(4, "tree", (0, 255, 0), "non-ground"),
RellisClass(5, "pole", (0, 153, 153), "non-ground"),
RellisClass(6, "water", (0, 128, 255), "ignore"),
RellisClass(7, "sky", (0, 0, 255), "ignore"),
RellisClass(8, "vehicle", (255, 255, 0), "non-ground"),
RellisClass(9, "object", (255, 0, 127), "ignore"),
RellisClass(10, "asphalt", (64, 64, 64), "ground"),
RellisClass(12, "building", (255, 0, 0), "non-ground"),
RellisClass(15, "log", (102, 0, 0), "non-ground"),
RellisClass(17, "person", (204, 153, 255), "non-ground"),
RellisClass(18, "fence", (102, 0, 204), "non-ground"),
RellisClass(19, "bush", (255, 153, 204), "non-ground"),
RellisClass(23, "concrete", (170, 170, 170), "ground"),
RellisClass(27, "barrier", (41, 121, 255), "non-ground"),
RellisClass(31, "puddle", (134, 255, 239), "ignore"),
RellisClass(33, "mud", (99, 66, 34), "ground"),
RellisClass(34, "rubble", (110, 22, 138), "non-ground"),
)
RELLIS_CLASS_BY_ID: Final = {item.label_id: item for item in RELLIS_CLASSES}
class RellisSmokeError(RuntimeError):
"""The pinned official example cannot satisfy the RELLIS smoke contract."""
def build_rellis_official_smoke_preview(
point_path: Path,
label_path: Path,
output_path: Path,
*,
preview_points: int = 20_000,
) -> dict[str, Any]:
"""Verify the official example and publish one bounded path-free preview."""
points_digest = _sha256_file(point_path)
labels_digest = _sha256_file(label_path)
if points_digest != RELLIS_EXAMPLE_POINTS_SHA256:
raise RellisSmokeError("RELLIS official example point digest is incompatible")
if labels_digest != RELLIS_EXAMPLE_LABELS_SHA256:
raise RellisSmokeError("RELLIS official example label digest is incompatible")
try:
frame = read_semantic_kitti_frame(point_path, label_path)
except DatasetFrameError as exc:
raise RellisSmokeError("RELLIS official example violates XYZI/label alignment") from exc
preview = rellis_native_scan_preview(
frame,
frame_id=RELLIS_EXAMPLE_FRAME_ID,
maximum_points=preview_points,
source_evidence={
"repository_url": RELLIS_REPOSITORY_URL,
"repository_commit": RELLIS_REPOSITORY_COMMIT,
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
"point_sha256": points_digest,
"label_sha256": labels_digest,
"license": RELLIS_LICENSE,
},
)
_atomic_json(output_path, preview)
return preview
def rellis_native_scan_preview(
frame: DatasetPointFrame,
*,
frame_id: str,
maximum_points: int,
source_evidence: dict[str, str],
) -> dict[str, Any]:
"""Build the common viewer artifact from a validated RELLIS native frame."""
if not frame_id or not 1 <= maximum_points <= MAX_RELLIS_PREVIEW_POINTS:
raise RellisSmokeError("RELLIS preview bounds are incompatible")
unknown = sorted(
int(value)
for value in np.unique(frame.semantic_labels)
if int(value) not in RELLIS_CLASS_BY_ID
)
if unknown:
raise RellisSmokeError("RELLIS frame contains labels absent from the pinned ontology")
required_evidence = {
"repository_url",
"repository_commit",
"label_config_sha256",
"point_sha256",
"label_sha256",
"license",
}
if set(source_evidence) != required_evidence or any(
not isinstance(value, str) or not value for value in source_evidence.values()
):
raise RellisSmokeError("RELLIS source evidence is incomplete")
sample_count = min(frame.point_count, maximum_points)
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
points = frame.points_xyz_m[indices]
remission = frame.remission[indices]
semantic = frame.semantic_labels[indices]
remission_min = float(np.min(remission))
remission_max = float(np.max(remission))
remission_span = remission_max - remission_min
if remission_span <= 0:
remission_u8 = np.zeros(sample_count, dtype=np.uint8)
else:
remission_u8 = np.rint(
(remission - remission_min) / remission_span * 255.0
).astype(np.uint8)
ground = np.zeros(sample_count, dtype=np.uint8)
evaluated = np.zeros(sample_count, dtype=np.uint8)
colors = np.zeros((sample_count, 3), dtype=np.uint8)
for label_id in np.unique(semantic):
item = RELLIS_CLASS_BY_ID[int(label_id)]
mask = semantic == label_id
colors[mask] = item.rgb
if item.ground_target != "ignore":
evaluated[mask] = 1
if item.ground_target == "ground":
ground[mask] = 1
present = Counter(int(value) for value in frame.semantic_labels)
present_classes = [
{
**RELLIS_CLASS_BY_ID[label_id].to_preview_dict(),
"source_point_count": count,
}
for label_id, count in sorted(present.items())
]
return {
"schema_version": RELLIS_PREVIEW_SCHEMA,
"source_id": RELLIS_SOURCE_ID,
"frame_id": frame_id,
"representation": "native-scan",
"sampling": "deterministic-even-index",
"source_point_count": frame.point_count,
"point_count": sample_count,
"points_xyz_m": points.tolist(),
"remission_0_to_255": remission_u8.tolist(),
"semantic_label_ids": semantic.astype(np.uint16).tolist(),
"semantic_rgb_0_to_255": colors.reshape(-1).tolist(),
"ground_truth_ground": ground.tolist(),
"evaluation_mask": evaluated.tolist(),
"classes": present_classes,
"coordinate_frame": {
"frame_id": "sensor/lidar/os1",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"transform_applied": False,
},
"ground_policy": {
"schema_version": RELLIS_GROUND_POLICY_SCHEMA,
"ground": [
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ground"
],
"non_ground": [
item.class_name
for item in RELLIS_CLASSES
if item.ground_target == "non-ground"
],
"ignore": [
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ignore"
],
},
"source_evidence": source_evidence,
"safety": {
"visualization_only": True,
"compatibility_smoke_only": True,
"navigation_or_safety_accepted": False,
},
}
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
try:
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024**2), b""):
digest.update(chunk)
except OSError as exc:
raise RellisSmokeError("RELLIS official example is unavailable") from exc
return digest.hexdigest()
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
mode="w",
dir=path.parent,
encoding="utf-8",
delete=False,
) as temporary:
temporary_path = Path(temporary.name)
json.dump(document, temporary, ensure_ascii=False, separators=(",", ":"))
temporary.flush()
os.fsync(temporary.fileno())
os.replace(temporary_path, path)

View File

@ -426,6 +426,9 @@ app.include_router(
dataset_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
),
dataset_rellis_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
),
dataset_ground_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
),

View File

@ -4,7 +4,7 @@ import os
import re
from collections.abc import Callable
from pathlib import Path
from typing import Any, Final
from typing import Annotated, Any, Final
from fastapi import APIRouter, HTTPException, Query, Response
@ -21,6 +21,7 @@ from k1link.compute import (
lidar_pack_detail,
)
from k1link.datasets import (
RELLIS_SOURCE_ID,
DatasetAdmissionError,
configured_dataset_admission_manifest,
configured_dataset_ground_preview,
@ -62,6 +63,7 @@ def build_lidar_router(
field_review_root_provider: RootProvider = configured_lidar_field_review_root,
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
) -> APIRouter:
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
@ -70,18 +72,32 @@ def build_lidar_router(
def get_dataset_gateway() -> dict[str, object]:
return dataset_gateway_catalog(
admission_manifest_path=dataset_admission_provider(),
rellis_preview_path=dataset_rellis_preview_provider(),
)
@router.get("/dataset-gateway/preview")
def get_dataset_gateway_preview() -> dict[str, Any]:
path = dataset_preview_provider()
def get_dataset_gateway_preview(
source_id: Annotated[
str,
Query(min_length=1, max_length=160),
] = "goose-3d/v2025-08-22",
) -> dict[str, Any]:
if source_id == "goose-3d/v2025-08-22":
path = dataset_preview_provider()
elif source_id == RELLIS_SOURCE_ID:
path = dataset_rellis_preview_provider()
else:
raise HTTPException(status_code=404, detail="Dataset source не найден.")
if path is None or not path.is_file():
raise HTTPException(
status_code=404,
detail="Первый размеченный native scan ещё не импортирован.",
detail="Размеченный native scan источника ещё не импортирован.",
)
try:
return read_dataset_native_scan_preview(path)
preview = read_dataset_native_scan_preview(path)
if preview["source_id"] != source_id:
raise DatasetAdmissionError("dataset preview source mismatch")
return preview
except DatasetAdmissionError as exc:
raise HTTPException(
status_code=500,

View File

@ -12,6 +12,7 @@ from fastapi.routing import APIRoute
from k1link.datasets import (
DatasetAdmissionError,
DatasetFrameError,
RELLIS_SOURCE_ID,
dataset_gateway_catalog,
goose_admission,
goose_benchmark,
@ -19,6 +20,7 @@ from k1link.datasets import (
read_dataset_ground_preview,
read_dataset_native_scan_preview,
read_semantic_kitti_frame,
rellis_native_scan_preview,
)
from k1link.datasets.goose_admission import admit_goose_validation
from k1link.datasets.goose_benchmark import (
@ -111,10 +113,14 @@ def test_dataset_gateway_api_is_read_only_and_path_free(
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
response = route() # type: ignore[operator]
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2"
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v3"
assert response["access"] == "read-only"
assert response["storage"]["admitted"] is True
assert response["storage"]["path_exposed"] is False
assert [source["source_id"] for source in response["sources"]] == [
"goose-3d/v2025-08-22",
RELLIS_SOURCE_ID,
]
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
response["storage"]["required_wsl_root"], # type: ignore[index]
"",
@ -286,6 +292,70 @@ def test_dataset_preview_api_is_read_only_and_integrity_checked(
assert response["frame_id"] == "frame-1"
def test_rellis_smoke_preview_keeps_semantickitti_alignment_and_ignore_policy(
tmp_path: Path,
) -> None:
points_path = tmp_path / "000104.bin"
labels_path = tmp_path / "000104.label"
np.asarray(
[
[1.0, 0.0, -1.0, 0.1],
[2.0, 0.2, -0.8, 0.2],
[3.0, -0.2, 0.5, 0.3],
[4.0, 0.0, -0.5, 0.4],
],
dtype="<f4",
).tofile(points_path)
np.asarray([3, 23, 4, 31], dtype="<u4").tofile(labels_path)
frame = read_semantic_kitti_frame(points_path, labels_path)
preview = rellis_native_scan_preview(
frame,
frame_id="000104",
maximum_points=4,
source_evidence={
"repository_url": "https://github.com/unmannedlab/RELLIS-3D",
"repository_commit": "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
"label_config_sha256": (
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
),
"point_sha256": (
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
),
"label_sha256": (
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
),
"license": "CC-BY-NC-SA-3.0",
},
)
preview_path = tmp_path / "rellis-preview.json"
preview_path.write_text(json.dumps(preview), encoding="utf-8")
decoded = read_dataset_native_scan_preview(preview_path)
assert decoded["source_id"] == RELLIS_SOURCE_ID
assert decoded["ground_truth_ground"] == [1, 1, 0, 0]
assert decoded["evaluation_mask"] == [1, 1, 1, 0]
assert decoded["coordinate_frame"]["x"] == "forward"
assert decoded["safety"]["compatibility_smoke_only"] is True
router = build_lidar_router(
dataset_admission_provider=lambda: None,
dataset_preview_provider=lambda: None,
dataset_rellis_preview_provider=lambda: preview_path,
)
route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview")
response = route(source_id=RELLIS_SOURCE_ID) # type: ignore[operator]
assert response["frame_id"] == "000104"
catalog = dataset_gateway_catalog(
Path("/mnt/d/NDC_MISSIONCORE/datasets"),
rellis_preview_path=preview_path,
)
rellis = catalog["sources"][1] # type: ignore[index]
assert rellis["admission"]["status"] == "smoke-ready" # type: ignore[index]
def test_goose_current_ground_is_scored_against_independent_labels(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,