feat: add RELLIS dataset compatibility smoke

This commit is contained in:
DCCONSTRUCTIONS 2026-07-25 20:45:30 +03:00
parent 13c35ff446
commit 054feec0d2
19 changed files with 1633 additions and 277 deletions

View File

@ -3,34 +3,38 @@ export type DatasetRepresentationId =
| "normalized-scan" | "normalized-scan"
| "rolling-local-map"; | "rolling-local-map";
export interface DatasetGatewayCatalog { export type DatasetSourceAdmissionStatus =
storage: {
configured: boolean;
admitted: boolean;
status: "ready" | "blocked-storage-policy";
attestation: "worker-manifest" | "local-worker-path" | "none";
manifestValid: boolean;
requiredWindowsRoot: string;
requiredWslRoot: string;
};
source: {
sourceId: string;
displayName: string;
role: string;
license: string;
format: string;
frameSemantics: "one-lidar-revolution";
platforms: string[];
superclasses: string[];
validationArchiveGb: number;
admissionStatus:
| "blocked-storage-policy" | "blocked-storage-policy"
| "ready-for-download" | "ready-for-download"
| "ready-for-smoke"
| "downloading" | "downloading"
| "downloaded" | "downloaded"
| "verifying" | "verifying"
| "verified" | "verified"
| "frame-ready"; | "frame-ready"
| "smoke-ready";
export interface DatasetSource {
sourceId: string;
sourceKind: "goose" | "rellis";
displayName: string;
role: string;
license: string;
commercialUse:
| "allowed-with-share-alike"
| "research-only-license-review-required";
format: string;
frameSemantics: "one-lidar-revolution";
sensor: string;
platforms: string[];
superclasses: string[];
smokeExampleMb: number | null;
primaryScanArchiveGb: number;
labelArchiveGb: number | null;
posesArchiveGb: number | null;
fragmentCount: number;
annotatedScanCount: number;
admissionStatus: DatasetSourceAdmissionStatus;
archive: { archive: {
bytesTransferred: number; bytesTransferred: number;
totalBytes: number; totalBytes: number;
@ -45,10 +49,22 @@ export interface DatasetGatewayCatalog {
semanticClassCount: number; semanticClassCount: number;
groundTruthGroundFraction: number; groundTruthGroundFraction: number;
previewPointCount: number; previewPointCount: number;
previewSha256: string; previewSha256: string | null;
previewAvailable: true; previewAvailable: true;
} | null; } | null;
}
export interface DatasetGatewayCatalog {
storage: {
configured: boolean;
admitted: boolean;
status: "ready" | "blocked-storage-policy";
attestation: "worker-manifest" | "local-worker-path" | "none";
manifestValid: boolean;
requiredWindowsRoot: string;
requiredWslRoot: string;
}; };
sources: DatasetSource[];
representations: Array<{ representations: Array<{
id: DatasetRepresentationId; id: DatasetRepresentationId;
title: string; title: string;
@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
} }
export interface DatasetNativeScanPreview { export interface DatasetNativeScanPreview {
sourceId: "goose-3d/v2025-08-22"; sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
frameId: string; frameId: string;
sourcePointCount: number; sourcePointCount: number;
pointCount: number; pointCount: number;
@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
semanticLabelIds: number[]; semanticLabelIds: number[];
semanticRgb0To255: number[]; semanticRgb0To255: number[];
groundTruthGround: number[]; groundTruthGround: number[];
evaluationMask: number[];
classes: Array<{ classes: Array<{
labelId: number; labelId: number;
className: string; className: string;
hex: string; hex: string;
challengeCategoryId: number; challengeCategoryId: number | null;
challengeCategoryName: string; challengeCategoryName: string | null;
groundTarget: "ground" | "non-ground" | "ignore" | null;
sourcePointCount: number | null;
}>; }>;
coordinateFrame: {
frameId: string;
x: "forward";
y: "left";
z: "up";
} | null;
compatibilitySmokeOnly: boolean;
} }
export interface DatasetGroundMetrics { export interface DatasetGroundMetrics {
@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
): DatasetGatewayCatalog { ): DatasetGatewayCatalog {
const source = record(value, "Dataset Gateway"); const source = record(value, "Dataset Gateway");
if ( if (
source.schema_version !== "missioncore.dataset-gateway-catalog/v2" source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
|| source.access !== "read-only" || source.access !== "read-only"
) { ) {
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим"); throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
@ -247,30 +273,47 @@ export function parseDatasetGatewayCatalog(
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение"); throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
} }
const sources = array(source.sources, "sources"); const sources = array(source.sources, "sources");
if (sources.length !== 1) { if (sources.length !== 2) {
throw new DatasetGatewayContractError("Ожидался один первичный dataset source"); throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
} }
const dataset = record(sources[0], "sources[0]"); const parsedSources = sources.map((value, index): DatasetSource => {
const download = record(dataset.download, "source.download"); const dataset = record(value, `sources[${index}]`);
const admission = record(dataset.admission, "source.admission"); const sourceKind = string(dataset.source_kind, "source_kind", true);
const sourceId = string(dataset.source_id, "source_id", true);
if (
(sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
&& (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
) {
throw new DatasetGatewayContractError("Dataset source identity неизвестна");
}
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
}
const download = record(dataset.download, `sources[${index}].download`);
const statistics = record(dataset.statistics, `sources[${index}].statistics`);
const admission = record(dataset.admission, `sources[${index}].admission`);
if (download.automatic !== false) { if (download.automatic !== false) {
throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически"); throw new DatasetGatewayContractError(
"Большой dataset нельзя загружать автоматически",
);
} }
const admissionStatus = admission.status; const admissionStatus = admission.status;
if ( if (
admissionStatus !== "ready-for-download" admissionStatus !== "ready-for-download"
&& admissionStatus !== "ready-for-smoke"
&& admissionStatus !== "blocked-storage-policy" && admissionStatus !== "blocked-storage-policy"
&& admissionStatus !== "downloading" && admissionStatus !== "downloading"
&& admissionStatus !== "downloaded" && admissionStatus !== "downloaded"
&& admissionStatus !== "verifying" && admissionStatus !== "verifying"
&& admissionStatus !== "verified" && admissionStatus !== "verified"
&& admissionStatus !== "frame-ready" && admissionStatus !== "frame-ready"
&& admissionStatus !== "smoke-ready"
) { ) {
throw new DatasetGatewayContractError("source admission status неизвестен"); throw new DatasetGatewayContractError("source admission status неизвестен");
} }
const archive = admission.archive === null const archive = admission.archive === null
? null ? null
: record(admission.archive, "source.admission.archive"); : record(admission.archive, `sources[${index}].admission.archive`);
const archiveValue = archive const archiveValue = archive
? { ? {
bytesTransferred: number( bytesTransferred: number(
@ -293,7 +336,7 @@ export function parseDatasetGatewayCatalog(
: null; : null;
const admittedFrame = admission.frame === null const admittedFrame = admission.frame === null
? null ? null
: record(admission.frame, "source.admission.frame"); : record(admission.frame, `sources[${index}].admission.frame`);
const frameValue = admittedFrame const frameValue = admittedFrame
? { ? {
frameId: string(admittedFrame.frame_id, "frame.frame_id", true), frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
@ -313,9 +356,7 @@ export function parseDatasetGatewayCatalog(
previewSha256: nullableDigest( previewSha256: nullableDigest(
admittedFrame.preview_sha256, admittedFrame.preview_sha256,
"frame.preview_sha256", "frame.preview_sha256",
) ?? (() => { ),
throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
})(),
previewAvailable: (() => { previewAvailable: (() => {
if (admittedFrame.preview_available !== true) { if (admittedFrame.preview_available !== true) {
throw new DatasetGatewayContractError( throw new DatasetGatewayContractError(
@ -326,6 +367,64 @@ export function parseDatasetGatewayCatalog(
})(), })(),
} }
: null; : null;
if (
sourceKind === "goose"
&& frameValue
&& frameValue.previewSha256 === null
) {
throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
}
const commercialUse = dataset.commercial_use;
if (
commercialUse !== "allowed-with-share-alike"
&& commercialUse !== "research-only-license-review-required"
) {
throw new DatasetGatewayContractError("Dataset license scope неизвестен");
}
return {
sourceId,
sourceKind,
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
commercialUse,
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
sensor: string(dataset.sensor, "sensor"),
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
smokeExampleMb: nullableNumber(
download.smoke_example_mb,
"smoke_example_mb",
),
primaryScanArchiveGb: number(
download.primary_scan_archive_gb,
"primary_scan_archive_gb",
),
labelArchiveGb: nullableNumber(
download.label_archive_gb,
"label_archive_gb",
),
posesArchiveGb: nullableNumber(
download.poses_archive_gb,
"poses_archive_gb",
),
fragmentCount: number(statistics.fragment_count, "fragment_count"),
annotatedScanCount: number(
statistics.annotated_scan_count,
"annotated_scan_count",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
};
});
if (
parsedSources[0]?.sourceKind !== "goose"
|| parsedSources[1]?.sourceKind !== "rellis"
) {
throw new DatasetGatewayContractError("Dataset source order несовместим");
}
const representations = array( const representations = array(
source.representations, source.representations,
"representations", "representations",
@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
) { ) {
throw new DatasetGatewayContractError("Vendor-map boundary завышен"); throw new DatasetGatewayContractError("Vendor-map boundary завышен");
} }
if (dataset.frame_semantics !== "one-lidar-revolution") {
throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
}
return { return {
storage: { storage: {
configured: boolean(storage.configured, "storage.configured"), configured: boolean(storage.configured, "storage.configured"),
@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
), ),
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"), requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
}, },
source: { sources: parsedSources,
sourceId: string(dataset.source_id, "source_id", true),
displayName: string(dataset.display_name, "display_name"),
role: string(dataset.role, "role", true),
license: string(dataset.license, "license"),
format: string(dataset.format, "format", true),
frameSemantics: "one-lidar-revolution",
platforms: displayStrings(dataset.platforms, "platforms"),
superclasses: strings(dataset.superclasses, "superclasses"),
validationArchiveGb: number(
download.validation_archive_gb,
"validation_archive_gb",
),
admissionStatus,
archive: archiveValue,
frame: frameValue,
},
representations, representations,
pipeline, pipeline,
currentInput: { currentInput: {
@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
value: unknown, value: unknown,
): DatasetNativeScanPreview { ): DatasetNativeScanPreview {
const source = record(value, "Dataset preview"); const source = record(value, "Dataset preview");
const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
&& source.source_id === "goose-3d/v2025-08-22";
const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
&& source.source_id === "rellis-3d/v1.1";
if ( if (
source.schema_version !== "missioncore.dataset-native-scan-preview/v1" (!isGoose && !isRellis)
|| source.source_id !== "goose-3d/v2025-08-22"
|| source.representation !== "native-scan" || source.representation !== "native-scan"
|| source.sampling !== "deterministic-even-index" || source.sampling !== "deterministic-even-index"
) { ) {
@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
"ground_truth_ground", "ground_truth_ground",
1, 1,
); );
const evaluationMask = isRellis
? integers(source.evaluation_mask, "evaluation_mask", 1)
: Array.from({ length: pointCount }, () => 1);
if ( if (
pointCount < 1 pointCount < 1
|| pointCount > 50_000 || pointCount > 50_000
@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
|| semanticLabelIds.length !== pointCount || semanticLabelIds.length !== pointCount
|| semanticRgb0To255.length !== pointCount * 3 || semanticRgb0To255.length !== pointCount * 3
|| groundTruthGround.length !== pointCount || groundTruthGround.length !== pointCount
|| evaluationMask.length !== pointCount
) { ) {
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены"); throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
} }
const classes = array(source.classes, "classes").map((value, index) => { const classes = array(source.classes, "classes").map((value, index) => {
const item = record(value, `classes[${index}]`); const item = record(value, `classes[${index}]`);
const groundTarget = item.ground_target;
if (
groundTarget !== undefined
&& groundTarget !== "ground"
&& groundTarget !== "non-ground"
&& groundTarget !== "ignore"
) {
throw new DatasetGatewayContractError("class.ground_target неизвестен");
}
const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
groundTarget === undefined ? null : groundTarget;
return { return {
labelId: number(item.label_id, "class.label_id"), labelId: number(item.label_id, "class.label_id"),
className: string(item.class_name, "class.class_name", true), className: string(item.class_name, "class.class_name", true),
hex: string(item.hex, "class.hex"), hex: string(item.hex, "class.hex"),
challengeCategoryId: number( challengeCategoryId: item.challenge_category_id === undefined
item.challenge_category_id, ? null
"class.challenge_category_id", : number(item.challenge_category_id, "class.challenge_category_id"),
), challengeCategoryName: item.challenge_category_name === undefined
challengeCategoryName: string( ? null
: string(
item.challenge_category_name, item.challenge_category_name,
"class.challenge_category_name", "class.challenge_category_name",
true, true,
), ),
groundTarget: normalizedGroundTarget,
sourcePointCount: item.source_point_count === undefined
? null
: number(item.source_point_count, "class.source_point_count"),
}; };
}); });
const safety = record(source.safety, "safety"); const safety = record(source.safety, "safety");
if ( if (
safety.visualization_only !== true safety.visualization_only !== true
|| safety.navigation_or_safety_accepted !== false || safety.navigation_or_safety_accepted !== false
|| (isRellis && safety.compatibility_smoke_only !== true)
) { ) {
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен"); throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
} }
const coordinateFrame = isRellis
? (() => {
const frame = record(source.coordinate_frame, "coordinate_frame");
if (
frame.frame_id !== "sensor/lidar/os1"
|| frame.x !== "forward"
|| frame.y !== "left"
|| frame.z !== "up"
|| frame.handedness !== "right"
|| frame.transform_applied !== false
) {
throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
}
return { return {
sourceId: "goose-3d/v2025-08-22", frameId: "sensor/lidar/os1",
x: "forward" as const,
y: "left" as const,
z: "up" as const,
};
})()
: null;
return {
sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
frameId: string(source.frame_id, "frame_id", true), frameId: string(source.frame_id, "frame_id", true),
sourcePointCount, sourcePointCount,
pointCount, pointCount,
@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
semanticLabelIds, semanticLabelIds,
semanticRgb0To255, semanticRgb0To255,
groundTruthGround, groundTruthGround,
evaluationMask,
classes, classes,
coordinateFrame,
compatibilitySmokeOnly: isRellis,
}; };
} }
@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
} }
export async function fetchDatasetNativeScanPreview( export async function fetchDatasetNativeScanPreview(
options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {}, options: {
sourceId?: DatasetNativeScanPreview["sourceId"];
signal?: AbortSignal;
fetcher?: DatasetFetch;
} = {},
): Promise<DatasetNativeScanPreview> { ): Promise<DatasetNativeScanPreview> {
const fetcher = options.fetcher ?? fetch; const fetcher = options.fetcher ?? fetch;
const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", { const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
const response = await fetcher(
`/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
{
method: "GET", method: "GET",
headers: { Accept: "application/json" }, headers: { Accept: "application/json" },
signal: options.signal, signal: options.signal,
}); },
);
return parseDatasetNativeScanPreview(await responseJson(response)); return parseDatasetNativeScanPreview(await responseJson(response));
} }

View File

@ -224,7 +224,9 @@
} }
.dataset-fragment-comparison, .dataset-fragment-comparison,
.dataset-fragment-risks { .dataset-fragment-risks,
.rellis-review__metrics,
.rellis-review__policy > div {
grid-template-columns: 1fr; grid-template-columns: 1fr;
} }

View File

@ -1223,6 +1223,85 @@
line-height: 1.5; line-height: 1.5;
} }
.rellis-review__metrics {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__metrics article {
display: grid;
gap: 0.2rem;
border-radius: 0.85rem;
background: var(--station-panel);
padding: 0.78rem 0.85rem;
}
.rellis-review__metrics span,
.rellis-review__metrics small {
color: var(--nodedc-text-muted);
font-size: 0.55rem;
}
.rellis-review__metrics strong {
color: var(--nodedc-text-primary);
font-size: 1.05rem;
}
.rellis-review__policy {
display: grid;
gap: 0.7rem;
border-radius: 1rem;
background: var(--station-panel);
padding: 0.9rem 1rem;
}
.rellis-review__policy h3,
.rellis-review__policy p {
margin: 0;
}
.rellis-review__policy h3 {
margin-top: 0.28rem;
color: var(--nodedc-text-primary);
font-size: 0.95rem;
}
.rellis-review__policy p {
max-width: 52rem;
margin-top: 0.3rem;
color: var(--nodedc-text-muted);
font-size: 0.6rem;
line-height: 1.5;
}
.rellis-review__policy > div {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.35rem;
}
.rellis-review__policy article {
display: grid;
align-content: start;
gap: 0.35rem;
border-radius: 0.75rem;
background: rgb(255 255 255 / 0.03);
padding: 0.7rem 0.75rem;
}
.rellis-review__policy article span {
color: var(--nodedc-text-muted);
font-size: 0.54rem;
}
.rellis-review__policy article strong {
color: var(--nodedc-text-secondary);
font-size: 0.62rem;
font-weight: 560;
line-height: 1.5;
}
.dataset-sequence-timeline { .dataset-sequence-timeline {
height: 2rem; height: 2rem;
} }
@ -2043,6 +2122,11 @@
font-size: 1rem; font-size: 1rem;
} }
.dataset-library__entries {
display: grid;
gap: 1rem;
}
.dataset-entry { .dataset-entry {
display: grid; display: grid;
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr); grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);

View File

@ -8,12 +8,15 @@ import {
import { import {
fetchDatasetGatewayCatalog, fetchDatasetGatewayCatalog,
type DatasetGatewayCatalog, type DatasetGatewayCatalog,
type DatasetSource,
type DatasetSourceAdmissionStatus,
} from "../core/lidar/datasetGateway"; } from "../core/lidar/datasetGateway";
import { import {
fetchPolygonRunCatalog, fetchPolygonRunCatalog,
type PolygonRunCatalog, type PolygonRunCatalog,
} from "../core/polygon/runArchive"; } from "../core/polygon/runArchive";
import { GooseDatasetReview } from "./GooseDatasetReview"; import { GooseDatasetReview } from "./GooseDatasetReview";
import { RellisDatasetReview } from "./RellisDatasetReview";
const representationLabels: Record<string, string> = { const representationLabels: Record<string, string> = {
"native-scan": "Исходный скан", "native-scan": "Исходный скан",
@ -27,15 +30,17 @@ function errorMessage(error: unknown): string {
: "Каталог датасетов недоступен."; : "Каталог датасетов недоступен.";
} }
function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) { function admissionLabel(status: DatasetSourceAdmissionStatus): string {
const labels: Record<typeof status, string> = { const labels: Record<DatasetSourceAdmissionStatus, string> = {
"blocked-storage-policy": "Worker не настроен", "blocked-storage-policy": "Worker не настроен",
"ready-for-download": "Готов к загрузке", "ready-for-download": "Готов к загрузке",
"ready-for-smoke": "Готов к smoke",
downloading: "Загружается", downloading: "Загружается",
downloaded: "Загружен", downloaded: "Загружен",
verifying: "Проверяется", verifying: "Проверяется",
verified: "Проверен", verified: "Проверен",
"frame-ready": "Первый кадр готов", "frame-ready": "Dataset готов",
"smoke-ready": "Smoke готов",
}; };
return labels[status]; return labels[status];
} }
@ -55,12 +60,42 @@ function isGooseQualificationRun(
&& item.scenarioGeneration.startsWith("goose-3d"); && item.scenarioGeneration.startsWith("goose-3d");
} }
function sourceReady(source: DatasetSource): boolean {
return source.admissionStatus === "frame-ready"
|| source.admissionStatus === "smoke-ready";
}
function sourceDescription(source: DatasetSource): string {
if (source.sourceKind === "goose") {
return "Восемь фрагментов и 961 разреженный размеченный оборот VLS-128. Здесь уже опубликовано покадровое сравнение текущего ground baseline и Patchwork++.";
}
return "Независимый off-road источник с Ouster OS1 64. Сейчас открыт официальный frame 000104 для проверки reader, осей, ontology и ground-policy; полный benchmark ещё не выполнялся.";
}
function sourceFooter(source: DatasetSource, storageReady: boolean): string {
if (source.admissionStatus === "frame-ready") {
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
}
if (source.admissionStatus === "smoke-ready") {
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
}
if (source.admissionStatus === "downloading") {
return "Архив остаётся на D worker; после загрузки проверяются digest, лицензия и point-label alignment.";
}
if (!storageReady) {
return "Сначала нужен допущенный Dataset Root на диске D worker.";
}
return source.sourceKind === "rellis"
? "Следующий шаг: пропустить официальный 24 МБ example через smoke admission."
: "Следующий шаг: загрузить validation archive на worker и проверить hash/license.";
}
export function DatasetGatewayWorkspace() { export function DatasetGatewayWorkspace() {
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null); const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
const [loading, setLoading] = useState(true); const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [reloadGeneration, setReloadGeneration] = useState(0); const [reloadGeneration, setReloadGeneration] = useState(0);
const [previewOpen, setPreviewOpen] = useState(false); const [openSourceId, setOpenSourceId] = useState<string | null>(null);
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null); const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
const [selectedRunId, setSelectedRunId] = useState<string | null>(null); const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
const [runError, setRunError] = useState<string | null>(null); const [runError, setRunError] = useState<string | null>(null);
@ -85,7 +120,9 @@ export function DatasetGatewayWorkspace() {
}, [reloadGeneration]); }, [reloadGeneration]);
useEffect(() => { useEffect(() => {
if (catalog?.source.admissionStatus !== "downloading") return; if (!catalog?.sources.some((source) => source.admissionStatus === "downloading")) {
return;
}
const timer = window.setTimeout( const timer = window.setTimeout(
() => setReloadGeneration((value) => value + 1), () => setReloadGeneration((value) => value + 1),
5_000, 5_000,
@ -113,10 +150,10 @@ export function DatasetGatewayWorkspace() {
return () => controller.abort(); return () => controller.abort();
}, []); }, []);
const downloadProgress = catalog?.source.archive
? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes
: null;
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? []; const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
const openSource = catalog?.sources.find(
(source) => source.sourceId === openSourceId,
) ?? null;
return ( return (
<div className="standard-workspace dataset-workspace"> <div className="standard-workspace dataset-workspace">
@ -125,17 +162,17 @@ export function DatasetGatewayWorkspace() {
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span> <span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
<h2>Независимая проверка алгоритмов</h2> <h2>Независимая проверка алгоритмов</h2>
<p> <p>
Датасет содержит известные размеченные данные. Мы показываем Каждый источник остаётся отдельным: собственные sensor domain,
конкретный фрагмент, результаты каждого алгоритма и ошибки разметка, лицензия и результаты. Совпадение результата на GOOSE и
относительно разметки. Он не подмешивается в диагностику реального RELLIS снижает риск подгонки под один датасет, но не заменяет replay
сенсора и не заменяет closed-loop симуляцию. реального сенсора и closed-loop симуляцию.
</p> </p>
</div> </div>
<ol aria-label="Как использовать датасет"> <ol aria-label="Как использовать датасет">
<li><span>01</span>Выбрать фрагмент</li> <li><span>01</span>Выбрать источник</li>
<li><span>02</span>Посмотреть сканы</li> <li><span>02</span>Проверить scan и labels</li>
<li><span>03</span>Сравнить алгоритмы</li> <li><span>03</span>Сравнить алгоритмы</li>
<li><span>04</span>Проверить риски</li> <li><span>04</span>Проверить переносимость</li>
</ol> </ol>
</section> </section>
@ -171,80 +208,91 @@ export function DatasetGatewayWorkspace() {
</StatusBadge> </StatusBadge>
</header> </header>
<article className="dataset-entry"> <div className="dataset-library__entries">
{catalog.sources.map((source) => {
const downloadProgress = source.archive
? source.archive.bytesTransferred / source.archive.totalBytes
: null;
const opened = source.sourceId === openSourceId;
return (
<article
className="dataset-entry"
data-source={source.sourceKind}
key={source.sourceId}
>
<div className="dataset-entry__identity"> <div className="dataset-entry__identity">
<span className="dataset-entry__mark">G</span> <span className="dataset-entry__mark">
{source.sourceKind === "goose" ? "G" : "R"}
</span>
<div> <div>
<span>PUBLIC · OFF-ROAD · LABELED</span> <span>
<h3>{catalog.source.displayName}</h3> PUBLIC · OFF-ROAD · LABELED
<p> {source.sourceKind === "rellis" ? " · RESEARCH" : ""}
Validation split: восемь фрагментов исходных записей и 961 </span>
разреженный размеченный оборот VLS-128. Это коллекция native <h3>{source.displayName}</h3>
scans для perception, а не непрерывное видео или траектория <p>{sourceDescription(source)}</p>
машины.
</p>
</div> </div>
</div> </div>
<dl> <dl>
<div> <div>
<dt>Состояние</dt> <dt>Состояние</dt>
<dd>{admissionLabel(catalog.source.admissionStatus)}</dd> <dd>{admissionLabel(source.admissionStatus)}</dd>
</div> </div>
<div> <div>
<dt>Validation</dt> <dt>Сканер</dt>
<dd>{catalog.source.validationArchiveGb} ГБ</dd> <dd>{source.sensor}</dd>
</div> </div>
<div> <div>
<dt>Группы</dt> <dt>Размеченных сканов</dt>
<dd>{catalog.source.superclasses.length} superclass</dd> <dd>{source.annotatedScanCount.toLocaleString("ru-RU")}</dd>
</div> </div>
<div> <div>
<dt>Лицензия</dt> <dt>Лицензия</dt>
<dd>{catalog.source.license}</dd> <dd>{source.license}</dd>
</div> </div>
</dl> </dl>
{catalog.source.admissionStatus === "downloading" {source.admissionStatus === "downloading"
&& catalog.source.archive && source.archive
&& downloadProgress !== null ? ( && downloadProgress !== null ? (
<div className="dataset-download" aria-label="Загрузка GOOSE validation"> <div
className="dataset-download"
aria-label={`Загрузка ${source.displayName}`}
>
<span style={{ width: `${downloadProgress * 100}%` }} /> <span style={{ width: `${downloadProgress * 100}%` }} />
<p> <p>
{formatBytes(catalog.source.archive.bytesTransferred)} {formatBytes(source.archive.bytesTransferred)}
{" из "} {" из "}
{formatBytes(catalog.source.archive.totalBytes)} {formatBytes(source.archive.totalBytes)}
</p> </p>
</div> </div>
) : null} ) : null}
<footer> <footer>
<p> <p>{sourceFooter(source, catalog.storage.admitted)}</p>
{catalog.source.admissionStatus === "frame-ready"
? "Native scan и разметка прошли admission. Можно открыть независимый ground truth."
: catalog.source.admissionStatus === "downloading"
? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment."
: catalog.storage.admitted
? "Следующий шаг: загрузить validation archive на worker и проверить hash/license."
: "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}
</p>
<div> <div>
<Button <Button
size="compact" size="compact"
variant="secondary" variant="secondary"
disabled={catalog.source.admissionStatus !== "frame-ready"} disabled={!sourceReady(source)}
title={ title={
catalog.source.admissionStatus === "frame-ready" sourceReady(source)
? "Открыть фрагменты, сканы и результаты анализа" ? "Открыть содержимое источника"
: "Размеченные сканы ещё не импортированы" : "Совместимый preview ещё не опубликован"
} }
onClick={() => setPreviewOpen((value) => !value)} onClick={() => setOpenSourceId(
opened ? null : source.sourceId,
)}
> >
{previewOpen ? "Закрыть датасет" : "Открыть датасет"} {opened ? "Закрыть датасет" : "Открыть датасет"}
</Button> </Button>
</div> </div>
</footer> </footer>
</article> </article>
);
})}
</div>
</GlassSurface> </GlassSurface>
{previewOpen && selectedRunId ? ( {openSource?.sourceKind === "goose" && selectedRunId ? (
<> <>
{gooseRuns.length > 1 ? ( {gooseRuns.length > 1 ? (
<div className="dataset-result-select"> <div className="dataset-result-select">
@ -267,12 +315,14 @@ export function DatasetGatewayWorkspace() {
) : null} ) : null}
<GooseDatasetReview runId={selectedRunId} /> <GooseDatasetReview runId={selectedRunId} />
</> </>
) : previewOpen ? ( ) : openSource?.sourceKind === "goose" ? (
<section className="polygon-review-unavailable"> <section className="polygon-review-unavailable">
<StatusBadge tone="warning">Review недоступен</StatusBadge> <StatusBadge tone="warning">Review недоступен</StatusBadge>
<h3>Датасет принят, но результаты анализа не опубликованы</h3> <h3>GOOSE принят, но анализ не опубликован</h3>
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p> <p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
</section> </section>
) : openSource?.sourceKind === "rellis" ? (
<RellisDatasetReview />
) : null} ) : null}
<details className="dataset-contract"> <details className="dataset-contract">

View File

@ -24,6 +24,7 @@ export interface LidarGroundPointCloudFrame {
disagreement: number[]; disagreement: number[];
candidateDisagreement?: number[]; candidateDisagreement?: number[];
groundTruthGround?: number[]; groundTruthGround?: number[];
evaluationMask?: number[];
}; };
} }
@ -87,6 +88,10 @@ function frameColors(
); );
} else if (mode === "ground-truth") { } else if (mode === "ground-truth") {
const groundTruth = frame.masks.groundTruthGround?.[index] === 1; const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
const evaluated = frame.masks.evaluationMask?.[index] !== 0;
if (!evaluated) {
setRgb(colors, offset, 0.86, 0.61, 0.24);
} else {
setRgb( setRgb(
colors, colors,
offset, offset,
@ -94,6 +99,7 @@ function frameColors(
groundTruth ? 0.86 : 0.33, groundTruth ? 0.86 : 0.33,
groundTruth ? 0.52 : 0.36, groundTruth ? 0.52 : 0.36,
); );
}
} else if (mode === "current") { } else if (mode === "current") {
setRgb( setRgb(
colors, colors,

View File

@ -0,0 +1,239 @@
import { useEffect, useMemo, useState } from "react";
import { StatusBadge } from "@nodedc/ui-react";
import {
fetchDatasetNativeScanPreview,
type DatasetNativeScanPreview,
} from "../core/lidar/datasetGateway";
import {
LidarGroundPointCloud,
type LidarGroundViewMode,
} from "./LidarGroundPointCloud";
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
["intensity", "Исходный скан"],
["semantic", "Классы RELLIS"],
["ground-truth", "Ground target"],
];
function formatPercent(value: number): string {
return new Intl.NumberFormat("ru-RU", {
style: "percent",
maximumFractionDigits: 1,
}).format(value);
}
function errorMessage(error: unknown): string {
return error instanceof Error && error.message.trim()
? error.message
: "Официальный пример RELLIS недоступен.";
}
function modeExplanation(mode: LidarGroundViewMode): string {
if (mode === "semantic") {
return "Цвета из закреплённой официальной ontology RELLIS-3D.";
}
if (mode === "ground-truth") {
return "Зелёный — ground, серый — non-ground, янтарный — ignore.";
}
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
}
export function RellisDatasetReview() {
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
const [error, setError] = useState<string | null>(null);
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
useEffect(() => {
const controller = new AbortController();
setError(null);
void fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
signal: controller.signal,
})
.then((value) => {
if (!controller.signal.aborted) setPreview(value);
})
.catch((loadError: unknown) => {
if (!controller.signal.aborted) setError(errorMessage(loadError));
});
return () => controller.abort();
}, []);
const scene = useMemo(() => {
if (!preview) return null;
const empty = Array.from({ length: preview.pointCount }, () => 0);
return {
pointCount: preview.pointCount,
pointsXyzM: preview.pointsXyzM,
intensity0To255: preview.remission0To255,
semanticRgb0To255: preview.semanticRgb0To255,
masks: {
currentGround: empty,
currentAssigned: preview.evaluationMask,
candidateGround: empty,
candidateAssigned: preview.evaluationMask,
disagreement: empty,
groundTruthGround: preview.groundTruthGround,
evaluationMask: preview.evaluationMask,
},
};
}, [preview]);
if (error) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="danger">Smoke preview недоступен</StatusBadge>
<h3>RELLIS-3D не открыт</h3>
<p>{error}</p>
</section>
);
}
if (!preview || !scene) {
return (
<section className="polygon-review-unavailable">
<StatusBadge tone="accent">Проверяем пример</StatusBadge>
<h3>Открываем официальный RELLIS scan</h3>
<p>Исходные dataset bytes не переносятся в браузер.</p>
</section>
);
}
const evaluatedPoints = preview.evaluationMask.reduce(
(total, value) => total + value,
0,
);
const groundPoints = preview.groundTruthGround.reduce(
(total, value, index) => total + value * (preview.evaluationMask[index] ?? 0),
0,
);
const groundClasses = preview.classes.filter(
(item) => item.groundTarget === "ground",
);
const nonGroundClasses = preview.classes.filter(
(item) => item.groundTarget === "non-ground",
);
const ignoredClasses = preview.classes.filter(
(item) => item.groundTarget === "ignore",
);
return (
<section
className="dataset-sequence-review rellis-review"
aria-label="Просмотр официального примера RELLIS-3D"
>
<header className="dataset-sequence-review__truth">
<div>
<span className="section-eyebrow">RELLIS S0 · COMPATIBILITY SMOKE</span>
<h2>Официальный Ouster scan читается без конвертации формата</h2>
<p>
Frame {preview.frameId}: исходные XYZI и point-wise labels остались
выровнены. Это проверка reader, осей, цветов и ground-policy ещё не
сравнение алгоритмов и не основание для навигации.
</p>
</div>
<dl>
<div>
<dt>Исходных точек</dt>
<dd>{preview.sourcePointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Browser preview</dt>
<dd>{preview.pointCount.toLocaleString("ru-RU")}</dd>
</div>
<div>
<dt>Sensor frame</dt>
<dd>{preview.coordinateFrame?.frameId ?? "—"}</dd>
</div>
</dl>
</header>
<section className="polygon-review-player">
<header className="polygon-review-player__header">
<div>
<span className="section-eyebrow">OFFICIAL EXAMPLE · FRAME 000104</span>
<strong>Ouster OS1 64 · X forward · Y left · Z up</strong>
</div>
<div className="polygon-review-mode" aria-label="Режим RELLIS preview">
{modes.map(([viewMode, label]) => (
<button
type="button"
key={viewMode}
data-active={mode === viewMode ? "true" : undefined}
onClick={() => setMode(viewMode)}
>
{label}
</button>
))}
</div>
</header>
<div className="polygon-review-stage">
<LidarGroundPointCloud
frame={scene}
mode={mode}
coordinateFrame="sensor-fixed"
/>
<div className="polygon-review-stage__legend">
<strong>{modes.find(([viewMode]) => viewMode === mode)?.[1]}</strong>
<span>{modeExplanation(mode)}</span>
</div>
</div>
</section>
<div className="rellis-review__metrics">
<article>
<span>Оценимых точек preview</span>
<strong>{formatPercent(evaluatedPoints / preview.pointCount)}</strong>
<small>ignore не участвует в метриках</small>
</article>
<article>
<span>Ground среди оценимых</span>
<strong>
{formatPercent(evaluatedPoints ? groundPoints / evaluatedPoints : 0)}
</strong>
<small>версионная mapping policy v1</small>
</article>
<article>
<span>Классов в этом scan</span>
<strong>{preview.classes.length}</strong>
<small>из 20 классов ontology</small>
</article>
</div>
<section className="rellis-review__policy" aria-label="Ground target mapping">
<header>
<span className="section-eyebrow">GROUND TARGET POLICY V1</span>
<h3>Что именно будет оцениваться</h3>
<p>
Mapping принадлежит Mission Core и версионируется отдельно от
ontology датасета. Вода, лужи, void, sky и неопределённый object не
засчитываются ни как ground, ни как obstacle.
</p>
</header>
<div>
<article>
<span>Ground</span>
<strong>{groundClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
<article>
<span>Non-ground</span>
<strong>
{nonGroundClasses.map((item) => item.className).join(" · ") || "—"}
</strong>
</article>
<article>
<span>Ignore</span>
<strong>{ignoredClasses.map((item) => item.className).join(" · ") || "—"}</strong>
</article>
</div>
</section>
<p className="dataset-sequence-review__raw-note">
Лицензия RELLIS-3D CC-BY-NC-SA-3.0: источник годится для независимой
исследовательской проверки, но коммерческое использование данных или
производных моделей требует отдельного лицензионного решения.
</p>
</section>
);
}

View File

@ -35,7 +35,7 @@ after(async () => {
function catalog(overrides = {}) { function catalog(overrides = {}) {
return { return {
schema_version: "missioncore.dataset-gateway-catalog/v2", schema_version: "missioncore.dataset-gateway-catalog/v3",
access: "read-only", access: "read-only",
storage: { storage: {
configured: false, configured: false,
@ -49,19 +49,60 @@ function catalog(overrides = {}) {
}, },
sources: [{ sources: [{
source_id: "goose-3d/v2025-08-22", source_id: "goose-3d/v2025-08-22",
source_kind: "goose",
display_name: "GOOSE 3D", display_name: "GOOSE 3D",
role: "primary-offroad-semantic-baseline", role: "primary-offroad-semantic-baseline",
license: "CC-BY-SA-4.0", license: "CC-BY-SA-4.0",
commercial_use: "allowed-with-share-alike",
format: "semantickitti-xyzi-label", format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution", frame_semantics: "one-lidar-revolution",
sensor: "VLS-128",
platforms: ["MuCAR-3", "ALICE", "Spot"], platforms: ["MuCAR-3", "ALICE", "Spot"],
annotations: ["semantic-point", "instance-point"], annotations: ["semantic-point", "instance-point"],
superclasses: ["natural-ground", "obstacle"], superclasses: ["natural-ground", "obstacle"],
download: { download: {
automatic: false, automatic: false,
reason: "operator-admitted-large-artifact-only", reason: "operator-admitted-large-artifact-only",
smoke_example_mb: null,
primary_scan_archive_gb: 3.3,
label_archive_gb: null,
poses_archive_gb: null,
validation_archive_gb: 3.3, validation_archive_gb: 3.3,
}, },
statistics: {
fragment_count: 8,
annotated_scan_count: 961,
},
admission: {
status: "blocked-storage-policy",
archive: null,
frame: null,
},
}, {
source_id: "rellis-3d/v1.1",
source_kind: "rellis",
display_name: "RELLIS-3D",
role: "independent-offroad-cross-dataset-check",
license: "CC-BY-NC-SA-3.0",
commercial_use: "research-only-license-review-required",
format: "semantickitti-xyzi-label",
frame_semantics: "one-lidar-revolution",
sensor: "Ouster OS1 64",
platforms: ["Clearpath Warthog"],
annotations: ["semantic-point"],
superclasses: ["ground", "obstacle", "ignore"],
download: {
automatic: false,
reason: "operator-admitted-large-artifact-only",
smoke_example_mb: 24,
primary_scan_archive_gb: 14,
label_archive_gb: 0.174,
poses_archive_gb: 0.174,
},
statistics: {
fragment_count: 5,
annotated_scan_count: 13_556,
},
admission: { admission: {
status: "blocked-storage-policy", status: "blocked-storage-policy",
archive: null, archive: null,
@ -115,7 +156,8 @@ test("parses three distinct LiDAR representations and current-input boundary", (
); );
assert.equal(parsed.representations[2].accumulation, true); assert.equal(parsed.representations[2].accumulation, true);
assert.equal(parsed.currentInput.admittedForPatchworkpp, false); assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
assert.equal(parsed.source.frameSemantics, "one-lidar-revolution"); assert.equal(parsed.sources[0].frameSemantics, "one-lidar-revolution");
assert.equal(parsed.sources[1].sourceKind, "rellis");
}); });
test("rejects path exposure and upgraded K1 semantics", () => { test("rejects path exposure and upgraded K1 semantics", () => {
@ -147,7 +189,7 @@ test("fetches the read-only gateway endpoint", async () => {
}); });
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway"); assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
assert.equal(calls[0].init.method, "GET"); assert.equal(calls[0].init.method, "GET");
assert.equal(parsed.source.displayName, "GOOSE 3D"); assert.equal(parsed.sources[0].displayName, "GOOSE 3D");
}); });
test("decodes one bounded point-aligned native-scan preview", async () => { test("decodes one bounded point-aligned native-scan preview", async () => {
@ -182,10 +224,19 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
assert.equal(parsed.pointCount, 2); assert.equal(parsed.pointCount, 2);
assert.deepEqual(parsed.groundTruthGround, [1, 0]); assert.deepEqual(parsed.groundTruthGround, [1, 0]);
let requestedUrl = null;
const fetched = await fetchDatasetNativeScanPreview({ const fetched = await fetchDatasetNativeScanPreview({
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }), fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
}); });
assert.equal(fetched.frameId, "frame-1"); assert.equal(fetched.frameId, "frame-1");
assert.deepEqual(fetched.evaluationMask, [1, 1]);
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=goose-3d%2Fv2025-08-22",
);
payload.semantic_rgb_0_to_255.pop(); payload.semantic_rgb_0_to_255.pop();
assert.throws( assert.throws(
@ -194,6 +245,92 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
); );
}); });
test("decodes the RELLIS compatibility smoke with explicit ignore mask", async () => {
const payload = {
schema_version: "missioncore.dataset-native-scan-preview/v2",
source_id: "rellis-3d/v1.1",
frame_id: "000104",
representation: "native-scan",
sampling: "deterministic-even-index",
source_point_count: 3,
point_count: 3,
points_xyz_m: [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
remission_0_to_255: [0, 127, 255],
semantic_label_ids: [3, 4, 31],
semantic_rgb_0_to_255: [0, 102, 0, 0, 255, 0, 239, 255, 134],
ground_truth_ground: [1, 0, 0],
evaluation_mask: [1, 1, 0],
classes: [
{
label_id: 3,
class_name: "grass",
hex: "#006600",
ground_target: "ground",
source_point_count: 1,
},
{
label_id: 4,
class_name: "tree",
hex: "#00ff00",
ground_target: "non-ground",
source_point_count: 1,
},
{
label_id: 31,
class_name: "puddle",
hex: "#efff86",
ground_target: "ignore",
source_point_count: 1,
},
],
coordinate_frame: {
frame_id: "sensor/lidar/os1",
handedness: "right",
x: "forward",
y: "left",
z: "up",
transform_applied: false,
},
ground_policy: {
schema_version: "missioncore.rellis-ground-target-policy/v1",
ground: ["grass"],
non_ground: ["tree"],
ignore: ["puddle"],
},
source_evidence: {
repository_url: "https://github.com/unmannedlab/RELLIS-3D",
repository_commit: "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
label_config_sha256: "5".repeat(64),
point_sha256: "e".repeat(64),
label_sha256: "9".repeat(64),
license: "CC-BY-NC-SA-3.0",
},
safety: {
visualization_only: true,
compatibility_smoke_only: true,
navigation_or_safety_accepted: false,
},
};
const parsed = parseDatasetNativeScanPreview(payload);
assert.equal(parsed.sourceId, "rellis-3d/v1.1");
assert.deepEqual(parsed.evaluationMask, [1, 1, 0]);
assert.equal(parsed.classes[2].groundTarget, "ignore");
assert.equal(parsed.coordinateFrame.frameId, "sensor/lidar/os1");
let requestedUrl = null;
await fetchDatasetNativeScanPreview({
sourceId: "rellis-3d/v1.1",
fetcher: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify(payload), { status: 200 });
},
});
assert.equal(
requestedUrl,
"/api/v1/lidar/dataset-gateway/preview?source_id=rellis-3d%2Fv1.1",
);
});
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => { test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
const payload = { const payload = {
schema_version: "missioncore.dataset-ground-comparison-preview/v2", schema_version: "missioncore.dataset-ground-comparison-preview/v2",

View File

@ -203,6 +203,14 @@ qualification bootstrap, never the product data plane. A shared
operator-facing deployment still requires authentication/RBAC, service operator-facing deployment still requires authentication/RBAC, service
supervision and an accepted routed worker transport. supervision and an accepted routed worker transport.
The Dataset Gateway is now multi-source. RELLIS S0 pins official repository
frame `000104`, verifies `131,072` Ouster XYZI points against aligned labels and
publishes a bounded semantic/ground-target preview beside GOOSE in the same
`Полигон → Датасеты` workflow. This closes only reader, axes, ontology,
licensing and versioned ignore-policy compatibility. Full RELLIS Ouster scans,
labels and poses remain D-only admission work before the cross-dataset
Current/Patchwork++ decision.
Mission Core remains a distributed web product. React is served by a long-lived Mission Core remains a distributed web product. React is served by a long-lived
Gateway/API and renders canonical state through browser-native components. Gateway/API and renders canonical state through browser-native components.
Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the

View File

@ -878,6 +878,11 @@ workflow:
- `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is - `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is
explicitly grouped into eight independent fragments; Play is bounded to one explicitly grouped into eight independent fragments; Play is bounded to one
fragment and described as accelerated review of sparse annotated scans. fragment and described as accelerated review of sparse annotated scans.
- The catalog is multi-source. RELLIS-3D v1.1 appears as an independent Ouster
domain with its own license and admission state. Its current `smoke-ready`
view shows official frame `000104`, semantic colors, FLU axes and the
versioned `ground / non-ground / ignore` mapping; it does not show invented
algorithm metrics.
- The dataset viewer exposes source frame number, nanosecond timestamp gaps, - The dataset viewer exposes source frame number, nanosecond timestamp gaps,
ground truth and Current/Patchwork++ overlays. ground truth and Current/Patchwork++ overlays.
- The fixed sensor-centric coordinate frame and operator camera are preserved - The fixed sensor-centric coordinate frame and operator camera are preserved
@ -894,6 +899,12 @@ The accepted `3.3 GB` annotated validation ZIP is not a continuous recording.
Continuous ego-motion requires a separately admitted raw ROS bag and Continuous ego-motion requires a separately admitted raw ROS bag and
localization source. localization source.
RELLIS S0 uses only a bounded derivative of the pinned official example for
browser inspection. The full `14 GB` Ouster SemanticKITTI scans, labels and
poses remain a separate D-only admission gate. Its `CC-BY-NC-SA-3.0` license
means cross-dataset research evidence cannot silently become an unrestricted
commercial runtime or training dependency.
The backend reads the configured repository from The backend reads the configured repository from
`MISSIONCORE_POLYGON_RUNS_ROOT` using `MISSIONCORE_POLYGON_RUNS_ROOT` using
`QualificationRunStore(read_only=True)`. It exposes only: `QualificationRunStore(read_only=True)`. It exposes only:

View File

@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
domain adaptation after a public baseline proves that the pipeline and metric domain adaptation after a public baseline proves that the pipeline and metric
harness work. harness work.
### L2.5 — Dataset Gateway — first-frame A/B complete ### L2.5 — Dataset Gateway — GOOSE qualification and RELLIS S0 complete
- [x] Define separate `native-scan`, `normalized-scan` and - [x] Define separate `native-scan`, `normalized-scan` and
`rolling-local-map` representations. `rolling-local-map` representations.
@ -317,12 +317,21 @@ harness work.
through an immutable R1 Polygon replay-shadow run. through an immutable R1 Polygon replay-shadow run.
- [x] Add named range/FOV/density/noise/dropout degradation profiles without - [x] Add named range/FOV/density/noise/dropout degradation profiles without
overwriting the native frame. overwriting the native frame.
- [x] Upgrade the gateway to two independently identified sources instead of
treating GOOSE as a universal dataset contract.
- [x] Pin and verify the official RELLIS-3D Ouster example, preserve its
`131,072` point/label alignment and publish a bounded semantic viewer.
- [x] Version the RELLIS `ground / non-ground / ignore` mapping and expose
licensing as research-only evidence.
- [ ] Admit the full RELLIS Ouster SemanticKITTI scans, labels and poses on
worker D, then run the same Current/Patchwork++ harness.
The architectural contract and run sequence are fixed in The architectural contract and run sequence are fixed in
`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its `docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its
published 3D format is one LiDAR revolution with point-wise semantic and published 3D format is one LiDAR revolution with point-wise semantic and
instance labels in off-road environments. RELLIS-3D remains the second-source instance labels in off-road environments. The RELLIS-3D compatibility smoke is
cross-check after the GOOSE harness is stable. now complete against official frame `000104`; the full second-source algorithm
cross-check remains the next gate.
### L3 — LiDAR-native 3D detection ### L3 — LiDAR-native 3D detection
@ -394,7 +403,10 @@ The near-term value is not a prettier point cloud:
The GOOSE validation-split and deterministic degradation gates are complete: The GOOSE validation-split and deterministic degradation gates are complete:
pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises
natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95 natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95
across 961 frames. The decision remains shadow-only. The highest-value across 961 frames. RELLIS S0 proves that the shared reader, axes, ontology and
immediate work is now a second-source cross-check plus LiDAR-native detection ground-target mapping work on a different Ouster off-road domain; it does not
on the same gateway. Nvblox and alternative SLAM remain later because their yet score an algorithm. The decision remains shadow-only. The highest-value
timing, pose and scan-geometry gates are not yet satisfied. immediate work is full RELLIS ground qualification, followed by the real
sensor input contract and LiDAR-native detection on the same gateway. Nvblox
and alternative SLAM remain later because their timing, pose and scan-geometry
gates are not yet satisfied.

View File

@ -19,12 +19,13 @@ datasets and future real onboard sensors.
Implemented now: Implemented now:
- `missioncore.dataset-gateway-catalog/v2`, exposed read-only at - multi-source `missioncore.dataset-gateway-catalog/v3`, exposed read-only at
`GET /api/v1/lidar/dataset-gateway`; `GET /api/v1/lidar/dataset-gateway`;
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a - path-free `missioncore.dataset-admission/v1` worker evidence instead of a
static React status; static React status;
- a bounded `missioncore.dataset-native-scan-preview/v1`, exposed read-only at - bounded `missioncore.dataset-native-scan-preview/v1` (GOOSE) and `/v2`
`GET /api/v1/lidar/dataset-gateway/preview`; (RELLIS smoke) artifacts, selected by source at
`GET /api/v1/lidar/dataset-gateway/preview?source_id=...`;
- explicit `native-scan`, `normalized-scan` and `rolling-local-map` - explicit `native-scan`, `normalized-scan` and `rolling-local-map`
representations; representations;
- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI - a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI
@ -39,6 +40,10 @@ Implemented now:
`Открыть` action; `Открыть` action;
- one admitted GOOSE validation frame with source colors, normalized remission - one admitted GOOSE validation frame with source colors, normalized remission
and an independent ground-truth view; and an independent ground-truth view;
- a pinned RELLIS-3D v1.1 compatibility smoke using the official Ouster example
frame `000104`, its native `131,072` XYZI points, point-aligned labels,
official ontology colors, explicit FLU sensor axes and a versioned
`ground / non-ground / ignore` evaluation policy;
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile: - a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
one-revolution scan and explicit absence of deskew/full vehicle TF claims; one-revolution scan and explicit absence of deskew/full vehicle TF claims;
@ -54,6 +59,8 @@ Not implemented:
- no implicit coordinate conversion; - no implicit coordinate conversion;
- no fake ring/timestamp reconstruction for K1 MQTT evidence; - no fake ring/timestamp reconstruction for K1 MQTT evidence;
- no model training or production promotion; - no model training or production promotion;
- no full RELLIS Ouster archive, pose set, algorithm comparison or ROS bag
admission yet;
- no rolling-map implementation yet. - no rolling-map implementation yet.
## Product surface boundary ## Product surface boundary
@ -73,10 +80,11 @@ The gateway is not part of device quality diagnostics.
backend evidence contract. They are not a second operator navigation item backend evidence contract. They are not a second operator navigation item
and are not presented as vehicle motion. and are not presented as vehicle motion.
Before real dataset bytes exist, the catalog explains the blocked storage gate Before compatible source evidence exists, the catalog explains the blocked
and keeps `Открыть датасет` disabled. It becomes available storage gate and keeps `Открыть датасет` disabled. GOOSE becomes available only
only after the worker manifest says `frame-ready` and the bounded preview passes after the worker manifest says `frame-ready`; RELLIS becomes available at
its own point-alignment and safety checks. `smoke-ready` only for the pinned official example. Both bounded previews must
pass their own identity, point-alignment and safety checks.
## Why public recordings look different ## Why public recordings look different
@ -191,8 +199,53 @@ time; TTL/dynamic filtering prevents stale ghosts.
- [x] Patchwork++ one-frame accuracy measured against ground truth. - [x] Patchwork++ one-frame accuracy measured against ground truth.
- [x] Current/Patchwork++ validation-split gate qualified. - [x] Current/Patchwork++ validation-split gate qualified.
- [x] Sensor-degradation matrix qualified. - [x] Sensor-degradation matrix qualified.
- [x] Dataset catalog accepts more than one independently identified source.
- [x] Official RELLIS Ouster example passes the shared SemanticKITTI reader.
- [x] RELLIS axes, ontology colors and versioned ground-target/ignore mapping
are visible in the same Dataset workflow.
- [ ] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
worker D.
- [ ] Current/Patchwork++ RELLIS comparison qualified without obstacle loss.
- [ ] Rolling local map with pose/TTL/dynamic policy qualified. - [ ] Rolling local map with pose/TTL/dynamic policy qualified.
## RELLIS S0 compatibility evidence
RELLIS is the independent second-source check, not extra decoration in the
catalog. GOOSE and RELLIS share the ingress format and viewer, but retain
separate source identity, sensor domain, ontology, licensing and results.
The pinned smoke input is the official repository example at commit
`c17a118fcaed1559f03cc32cc3a91dedc557f8b8`:
- point file: `utils/example/000104.bin`;
- label file: `utils/example/000104.label`;
- source points: `131,072`;
- point SHA-256:
`ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af`;
- label SHA-256:
`9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e`;
- official ontology config SHA-256:
`573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad`;
- bounded browser preview: `20,000` deterministic even-index points;
- coordinate frame: right-handed, `x` forward, `y` left, `z` up; no vehicle
transform, pose registration or deskew is claimed;
- license: `CC-BY-NC-SA-3.0`; this is a research qualification source, not an
unrestricted commercial runtime dependency.
Mission Core ground policy v1 treats `dirt`, `grass`, `asphalt`, `concrete` and
`mud` as ground; clear structures, vegetation, people, vehicles and rubble as
non-ground; and `void`, `water`, `sky`, generic `object` and `puddle` as
`ignore`. The ignore set is excluded from future metrics instead of being
silently counted as either free ground or obstacle. This mapping is a Mission
Core evaluation decision, not an upstream RELLIS claim.
The smoke closes reader, alignment, axes, palette and mapping compatibility
only. The next gate admits the `14 GB` Ouster SemanticKITTI scans plus the
`174 MB` labels and `174 MB` poses to D-only worker storage. Full ROS bags are
not needed for the ground cross-check. Patchwork++ may be compared only after a
physical Ouster mounting-height profile is evidenced; no height is inferred
from the example cloud.
## First GOOSE admission evidence ## First GOOSE admission evidence
- worker root: canonical D-only root, path never exposed by the HTTP API; - worker root: canonical D-only root, path never exposed by the HTTP API;

View File

@ -39,6 +39,12 @@ first admitted source because it publishes off-road point-wise semantic and
instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its
annotated point-cloud file represents one LiDAR revolution. annotated point-cloud file represents one LiDAR revolution.
The gateway contract is multi-source, not GOOSE-shaped. RELLIS-3D v1.1 is the
second source: it uses the same lossless SemanticKITTI ingress but keeps a
separate Ouster OS1 domain, ontology, admission state, ground-target mapping,
license and result set. Shared format never implies shared calibration,
mounting height or commercial-use rights.
The gateway: The gateway:
- preserves the native GOOSE frame before adaptation; - preserves the native GOOSE frame before adaptation;
@ -48,6 +54,10 @@ The gateway:
- refuses automatic downloads of large archives; - refuses automatic downloads of large archives;
- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror; - admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror;
- never promotes K1 `lio_pcl` to `native-scan`. - never promotes K1 `lio_pcl` to `native-scan`.
- excludes explicitly ambiguous RELLIS classes through a versioned `ignore`
mask instead of silently scoring them as ground or obstacle;
- exposes the RELLIS `CC-BY-NC-SA-3.0` restriction as research evidence and
never treats it as an unrestricted production dependency.
Large-artifact execution remains worker-local. The browser receives only a Large-artifact execution remains worker-local. The browser receives only a
path-free admission manifest and a bounded visualization preview. The canonical path-free admission manifest and a bounded visualization preview. The canonical
@ -82,6 +92,8 @@ unavailable rather than inventing timestamps.
- Until bytes are installed, the UI exposes an explicit empty state. After a - Until bytes are installed, the UI exposes an explicit empty state. After a
`frame-ready` worker admission, the same surface becomes an interactive `frame-ready` worker admission, the same surface becomes an interactive
source/remission/ground-truth viewer without changing the representation. source/remission/ground-truth viewer without changing the representation.
- A bounded `smoke-ready` source may prove reader, axes, ontology and mapping
compatibility without claiming full-dataset or algorithm qualification.
- Dataset and device inputs can share downstream algorithms only after their - Dataset and device inputs can share downstream algorithms only after their
normalized contracts match. normalized contracts match.
- Sensor adaptation may change range, FOV, point density, noise and dropout for - Sensor adaptation may change range, FOV, point density, noise and dropout for
@ -115,6 +127,7 @@ unavailable rather than inventing timestamps.
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/) - [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788) - [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1) - [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
- [RELLIS-3D official repository](https://github.com/unmannedlab/RELLIS-3D)
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2) - [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox) - [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html) - [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)

View File

@ -38,6 +38,15 @@ from k1link.datasets.goose_qualification import (
GroundAcceptancePolicy, GroundAcceptancePolicy,
qualify_goose_ground, qualify_goose_ground,
) )
from k1link.datasets.rellis_smoke import (
RELLIS_CLASSES,
RELLIS_GROUND_POLICY_SCHEMA,
RELLIS_PREVIEW_SCHEMA,
RELLIS_SOURCE_ID,
RellisSmokeError,
build_rellis_official_smoke_preview,
rellis_native_scan_preview,
)
__all__ = [ __all__ = [
"DATASET_GATEWAY_CATALOG_SCHEMA", "DATASET_GATEWAY_CATALOG_SCHEMA",
@ -54,12 +63,18 @@ __all__ = [
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA", "GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
"GOOSE_QUALIFICATION_PROFILE_SCHEMA", "GOOSE_QUALIFICATION_PROFILE_SCHEMA",
"GOOSE_QUALIFICATION_REPORT_SCHEMA", "GOOSE_QUALIFICATION_REPORT_SCHEMA",
"RELLIS_CLASSES",
"RELLIS_GROUND_POLICY_SCHEMA",
"RELLIS_PREVIEW_SCHEMA",
"RELLIS_SOURCE_ID",
"RellisSmokeError",
"GoosePatchworkProfile", "GoosePatchworkProfile",
"GroundAcceptancePolicy", "GroundAcceptancePolicy",
"DegradationProfile", "DegradationProfile",
"DEFAULT_DEGRADATIONS", "DEFAULT_DEGRADATIONS",
"benchmark_goose_current_ground", "benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground", "benchmark_goose_patchwork_ground",
"build_rellis_official_smoke_preview",
"configured_dataset_admission_manifest", "configured_dataset_admission_manifest",
"configured_dataset_ground_preview", "configured_dataset_ground_preview",
"configured_dataset_preview", "configured_dataset_preview",
@ -69,5 +84,6 @@ __all__ = [
"read_dataset_ground_preview", "read_dataset_ground_preview",
"read_dataset_native_scan_preview", "read_dataset_native_scan_preview",
"read_semantic_kitti_frame", "read_semantic_kitti_frame",
"rellis_native_scan_preview",
"qualify_goose_ground", "qualify_goose_ground",
] ]

View File

@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
) )
from k1link.datasets.goose_qualification import qualify_goose_ground from k1link.datasets.goose_qualification import qualify_goose_ground
from k1link.datasets.goose_review import build_goose_ground_review_pack from k1link.datasets.goose_review import build_goose_ground_review_pack
from k1link.datasets.rellis_smoke import (
RellisSmokeError,
build_rellis_official_smoke_preview,
)
app = typer.Typer( app = typer.Typer(
add_completion=False, add_completion=False,
@ -158,5 +162,50 @@ def build_goose_ground_review_command(
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True)) typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
@app.command("build-rellis-smoke-preview")
def build_rellis_smoke_preview_command(
points: Annotated[
Path,
typer.Option("--points", exists=True, dir_okay=False, resolve_path=True),
],
labels: Annotated[
Path,
typer.Option("--labels", exists=True, dir_okay=False, resolve_path=True),
],
out: Annotated[
Path,
typer.Option("--out", dir_okay=False, resolve_path=True),
],
preview_points: Annotated[
int,
typer.Option("--preview-points", min=1, max=50_000),
] = 20_000,
) -> None:
"""Verify the pinned official RELLIS example and publish a bounded preview."""
try:
preview = build_rellis_official_smoke_preview(
points,
labels,
out,
preview_points=preview_points,
)
except RellisSmokeError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(
json.dumps(
{
"source_id": preview["source_id"],
"frame_id": preview["frame_id"],
"source_point_count": preview["source_point_count"],
"preview_point_count": preview["point_count"],
},
ensure_ascii=False,
sort_keys=True,
)
)
if __name__ == "__main__": if __name__ == "__main__":
app() app()

View File

@ -18,7 +18,7 @@ import numpy.typing as npt
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2" DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v3"
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1" DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1" DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2" DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
@ -177,6 +177,7 @@ def _is_worker_d_storage(root: Path | None) -> bool:
def dataset_gateway_catalog( def dataset_gateway_catalog(
dataset_root: Path | None = None, dataset_root: Path | None = None,
admission_manifest_path: Path | None = None, admission_manifest_path: Path | None = None,
rellis_preview_path: Path | None = None,
) -> dict[str, object]: ) -> dict[str, object]:
"""Return the path-free, read-only ingress plan and current admission state.""" """Return the path-free, read-only ingress plan and current admission state."""
@ -196,6 +197,14 @@ def dataset_gateway_catalog(
locally_admitted = _is_worker_d_storage(root) locally_admitted = _is_worker_d_storage(root)
worker_admitted = bool(admission and admission["storage"]["admitted"]) worker_admitted = bool(admission and admission["storage"]["admitted"])
storage_admitted = locally_admitted or worker_admitted storage_admitted = locally_admitted or worker_admitted
rellis_preview: dict[str, Any] | None = None
if rellis_preview_path is not None and rellis_preview_path.is_file():
try:
candidate = read_dataset_native_scan_preview(rellis_preview_path)
if candidate.get("source_id") == "rellis-3d/v1.1":
rellis_preview = candidate
except DatasetAdmissionError:
pass
source_status = ( source_status = (
str(admission["status"]) str(admission["status"])
if admission is not None if admission is not None
@ -212,6 +221,20 @@ def dataset_gateway_catalog(
if storage_admitted if storage_admitted
else "configure-dataset-root-on-worker-d" else "configure-dataset-root-on-worker-d"
) )
rellis_status = (
"smoke-ready"
if rellis_preview is not None
else "ready-for-smoke"
if storage_admitted
else "blocked-storage-policy"
)
catalog_next_action = (
"admit-rellis-ouster-semantickitti-to-worker-d"
if rellis_preview is not None and source_status == "frame-ready"
else "build-rellis-official-example-smoke"
if source_status == "frame-ready"
else next_action
)
return { return {
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA, "schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
"access": "read-only", "access": "read-only",
@ -234,11 +257,14 @@ def dataset_gateway_catalog(
"sources": [ "sources": [
{ {
"source_id": "goose-3d/v2025-08-22", "source_id": "goose-3d/v2025-08-22",
"source_kind": "goose",
"display_name": "GOOSE 3D", "display_name": "GOOSE 3D",
"role": "primary-offroad-semantic-baseline", "role": "primary-offroad-semantic-baseline",
"license": "CC-BY-SA-4.0", "license": "CC-BY-SA-4.0",
"commercial_use": "allowed-with-share-alike",
"format": "semantickitti-xyzi-label", "format": "semantickitti-xyzi-label",
"frame_semantics": "one-lidar-revolution", "frame_semantics": "one-lidar-revolution",
"sensor": "VLS-128",
"platforms": ["MuCAR-3", "ALICE", "Spot"], "platforms": ["MuCAR-3", "ALICE", "Spot"],
"annotations": ["semantic-point", "instance-point"], "annotations": ["semantic-point", "instance-point"],
"superclasses": [ "superclasses": [
@ -255,10 +281,18 @@ def dataset_gateway_catalog(
"download": { "download": {
"automatic": False, "automatic": False,
"reason": "operator-admitted-large-artifact-only", "reason": "operator-admitted-large-artifact-only",
"smoke_example_mb": None,
"primary_scan_archive_gb": 3.3,
"label_archive_gb": None,
"poses_archive_gb": None,
"training_archive_gb": 27.0, "training_archive_gb": 27.0,
"validation_archive_gb": 3.3, "validation_archive_gb": 3.3,
"test_archive_gb": 3.3, "test_archive_gb": 3.3,
}, },
"statistics": {
"fragment_count": 8,
"annotated_scan_count": 961,
},
"admission": { "admission": {
"status": source_status, "status": source_status,
"native_scan": "ready-after-download", "native_scan": "ready-after-download",
@ -267,7 +301,72 @@ def dataset_gateway_catalog(
"archive": admission["archive"] if admission is not None else None, "archive": admission["archive"] if admission is not None else None,
"frame": admission["frame"] if admission is not None else None, "frame": admission["frame"] if admission is not None else None,
}, },
},
{
"source_id": "rellis-3d/v1.1",
"source_kind": "rellis",
"display_name": "RELLIS-3D",
"role": "independent-offroad-cross-dataset-check",
"license": "CC-BY-NC-SA-3.0",
"commercial_use": "research-only-license-review-required",
"format": "semantickitti-xyzi-label",
"frame_semantics": "one-lidar-revolution",
"sensor": "Ouster OS1 64",
"platforms": ["Clearpath Warthog"],
"annotations": ["semantic-point"],
"superclasses": [
"ground",
"vegetation",
"structure",
"obstacle",
"vehicle",
"human",
"water",
"ignore",
],
"download": {
"automatic": False,
"reason": "operator-admitted-large-artifact-only",
"smoke_example_mb": 24.0,
"primary_scan_archive_gb": 14.0,
"label_archive_gb": 0.174,
"poses_archive_gb": 0.174,
"training_archive_gb": None,
"validation_archive_gb": None,
"test_archive_gb": None,
},
"statistics": {
"fragment_count": 5,
"annotated_scan_count": 13_556,
},
"admission": {
"status": rellis_status,
"native_scan": (
"official-example-compatible"
if rellis_preview is not None
else "requires-official-example-smoke"
),
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
"rolling_local_map": "requires-poses-timing-and-map-policy",
"archive": None,
"frame": (
{
"frame_id": rellis_preview["frame_id"],
"point_count": rellis_preview["source_point_count"],
"semantic_class_count": len(rellis_preview["classes"]),
"ground_truth_ground_fraction": (
sum(rellis_preview["ground_truth_ground"])
/ rellis_preview["point_count"]
),
"preview_point_count": rellis_preview["point_count"],
"preview_sha256": None,
"preview_available": True,
} }
if rellis_preview is not None
else None
),
},
},
], ],
"representations": [ "representations": [
{ {
@ -330,7 +429,7 @@ def dataset_gateway_catalog(
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan", "reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
} }
], ],
"next_action": next_action, "next_action": catalog_next_action,
} }
@ -417,9 +516,13 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]: def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview") document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
identity = (document.get("schema_version"), document.get("source_id"))
if ( if (
document.get("schema_version") != DATASET_PREVIEW_SCHEMA identity
or document.get("source_id") != "goose-3d/v2025-08-22" not in {
(DATASET_PREVIEW_SCHEMA, "goose-3d/v2025-08-22"),
("missioncore.dataset-native-scan-preview/v2", "rellis-3d/v1.1"),
}
or document.get("representation") != "native-scan" or document.get("representation") != "native-scan"
or document.get("sampling") != "deterministic-even-index" or document.get("sampling") != "deterministic-even-index"
): ):
@ -433,6 +536,11 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
semantic_ids = document.get("semantic_label_ids") semantic_ids = document.get("semantic_label_ids")
semantic_rgb = document.get("semantic_rgb_0_to_255") semantic_rgb = document.get("semantic_rgb_0_to_255")
ground = document.get("ground_truth_ground") ground = document.get("ground_truth_ground")
evaluated = (
document.get("evaluation_mask")
if identity[1] == "rellis-3d/v1.1"
else [1] * point_count
)
if ( if (
not isinstance(points, list) not isinstance(points, list)
or len(points) != point_count or len(points) != point_count
@ -444,6 +552,8 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
or len(semantic_rgb) != point_count * 3 or len(semantic_rgb) != point_count * 3
or not isinstance(ground, list) or not isinstance(ground, list)
or len(ground) != point_count or len(ground) != point_count
or not isinstance(evaluated, list)
or len(evaluated) != point_count
): ):
raise DatasetAdmissionError("dataset preview arrays are not point-aligned") raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
for point in points: for point in points:
@ -463,6 +573,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
(semantic_ids, 65_535, "semantic label"), (semantic_ids, 65_535, "semantic label"),
(semantic_rgb, 255, "semantic color"), (semantic_rgb, 255, "semantic color"),
(ground, 1, "ground mask"), (ground, 1, "ground mask"),
(evaluated, 1, "evaluation mask"),
): ):
if any( if any(
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
@ -473,16 +584,81 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
if not isinstance(classes, list) or len(classes) > 64: if not isinstance(classes, list) or len(classes) > 64:
raise DatasetAdmissionError("dataset preview class catalog is incompatible") raise DatasetAdmissionError("dataset preview class catalog is incompatible")
safety = _object(document.get("safety"), "safety") safety = _object(document.get("safety"), "safety")
if safety != { expected_safety = {
"visualization_only": True, "visualization_only": True,
"navigation_or_safety_accepted": False, "navigation_or_safety_accepted": False,
}: }
if identity[1] == "rellis-3d/v1.1":
expected_safety["compatibility_smoke_only"] = True
if safety != expected_safety:
raise DatasetAdmissionError("dataset preview safety boundary is incompatible") raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]: if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
raise DatasetAdmissionError("dataset preview frame id is incompatible") raise DatasetAdmissionError("dataset preview frame id is incompatible")
if identity[1] == "rellis-3d/v1.1":
_validate_rellis_preview_metadata(document, classes)
return document return document
def _validate_rellis_preview_metadata(
document: dict[str, Any],
classes: list[Any],
) -> None:
coordinate_frame = _object(document.get("coordinate_frame"), "coordinate_frame")
if coordinate_frame != {
"frame_id": "sensor/lidar/os1",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"transform_applied": False,
}:
raise DatasetAdmissionError("RELLIS coordinate frame is incompatible")
policy = _object(document.get("ground_policy"), "ground_policy")
if (
policy.get("schema_version") != "missioncore.rellis-ground-target-policy/v1"
or set(policy) != {"schema_version", "ground", "non_ground", "ignore"}
or any(
not isinstance(policy.get(key), list)
or any(not isinstance(value, str) or not value for value in policy[key])
for key in ("ground", "non_ground", "ignore")
)
):
raise DatasetAdmissionError("RELLIS ground target policy is incompatible")
for item in classes:
value = _object(item, "class")
if (
value.get("ground_target") not in {"ground", "non-ground", "ignore"}
or not isinstance(value.get("label_id"), int)
or not isinstance(value.get("class_name"), str)
or not isinstance(value.get("hex"), str)
or not isinstance(value.get("source_point_count"), int)
):
raise DatasetAdmissionError("RELLIS class catalog is incompatible")
evidence = _object(document.get("source_evidence"), "source_evidence")
if (
set(evidence)
!= {
"repository_url",
"repository_commit",
"label_config_sha256",
"point_sha256",
"label_sha256",
"license",
}
or evidence.get("repository_url") != "https://github.com/unmannedlab/RELLIS-3D"
or evidence.get("repository_commit")
!= "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
or evidence.get("label_config_sha256")
!= "573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
or evidence.get("point_sha256")
!= "ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
or evidence.get("label_sha256")
!= "9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
or evidence.get("license") != "CC-BY-NC-SA-3.0"
):
raise DatasetAdmissionError("RELLIS source evidence is incompatible")
def read_dataset_ground_preview(path: Path) -> dict[str, Any]: def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview") document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
if ( if (

View File

@ -0,0 +1,275 @@
"""Pinned RELLIS-3D compatibility smoke for the shared Dataset Gateway.
The smoke check deliberately stops before algorithm qualification. It proves
that Mission Core can read one official Ouster OS1 SemanticKITTI frame, preserve
point/label alignment, apply an explicit ground-evaluation policy and publish a
bounded visualization artifact. Full RELLIS archives and ROS bags remain
worker-D-only inputs.
"""
from __future__ import annotations
import hashlib
import json
import os
import tempfile
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final, Literal
import numpy as np
from k1link.datasets.gateway import DatasetFrameError, DatasetPointFrame, read_semantic_kitti_frame
RELLIS_SOURCE_ID: Final = "rellis-3d/v1.1"
RELLIS_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v2"
RELLIS_GROUND_POLICY_SCHEMA: Final = "missioncore.rellis-ground-target-policy/v1"
RELLIS_REPOSITORY_URL: Final = "https://github.com/unmannedlab/RELLIS-3D"
RELLIS_REPOSITORY_COMMIT: Final = "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
RELLIS_LABEL_CONFIG_SHA256: Final = (
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
)
RELLIS_EXAMPLE_POINTS_SHA256: Final = (
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
)
RELLIS_EXAMPLE_LABELS_SHA256: Final = (
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
)
RELLIS_EXAMPLE_FRAME_ID: Final = "000104"
RELLIS_LICENSE: Final = "CC-BY-NC-SA-3.0"
MAX_RELLIS_PREVIEW_POINTS: Final = 50_000
GroundTarget = Literal["ground", "non-ground", "ignore"]
@dataclass(frozen=True)
class RellisClass:
label_id: int
class_name: str
bgr: tuple[int, int, int]
ground_target: GroundTarget
@property
def rgb(self) -> tuple[int, int, int]:
blue, green, red = self.bgr
return red, green, blue
@property
def hex(self) -> str:
red, green, blue = self.rgb
return f"#{red:02x}{green:02x}{blue:02x}"
def to_preview_dict(self) -> dict[str, object]:
return {
"label_id": self.label_id,
"class_name": self.class_name,
"hex": self.hex,
"ground_target": self.ground_target,
}
# IDs, names and BGR colors are pinned to the official repository config at
# RELLIS_REPOSITORY_COMMIT. The final column is Mission Core's versioned,
# algorithm-scoped ground evaluation policy; it is not an upstream claim.
RELLIS_CLASSES: Final[tuple[RellisClass, ...]] = (
RellisClass(0, "void", (0, 0, 0), "ignore"),
RellisClass(1, "dirt", (108, 64, 20), "ground"),
RellisClass(3, "grass", (0, 102, 0), "ground"),
RellisClass(4, "tree", (0, 255, 0), "non-ground"),
RellisClass(5, "pole", (0, 153, 153), "non-ground"),
RellisClass(6, "water", (0, 128, 255), "ignore"),
RellisClass(7, "sky", (0, 0, 255), "ignore"),
RellisClass(8, "vehicle", (255, 255, 0), "non-ground"),
RellisClass(9, "object", (255, 0, 127), "ignore"),
RellisClass(10, "asphalt", (64, 64, 64), "ground"),
RellisClass(12, "building", (255, 0, 0), "non-ground"),
RellisClass(15, "log", (102, 0, 0), "non-ground"),
RellisClass(17, "person", (204, 153, 255), "non-ground"),
RellisClass(18, "fence", (102, 0, 204), "non-ground"),
RellisClass(19, "bush", (255, 153, 204), "non-ground"),
RellisClass(23, "concrete", (170, 170, 170), "ground"),
RellisClass(27, "barrier", (41, 121, 255), "non-ground"),
RellisClass(31, "puddle", (134, 255, 239), "ignore"),
RellisClass(33, "mud", (99, 66, 34), "ground"),
RellisClass(34, "rubble", (110, 22, 138), "non-ground"),
)
RELLIS_CLASS_BY_ID: Final = {item.label_id: item for item in RELLIS_CLASSES}
class RellisSmokeError(RuntimeError):
"""The pinned official example cannot satisfy the RELLIS smoke contract."""
def build_rellis_official_smoke_preview(
point_path: Path,
label_path: Path,
output_path: Path,
*,
preview_points: int = 20_000,
) -> dict[str, Any]:
"""Verify the official example and publish one bounded path-free preview."""
points_digest = _sha256_file(point_path)
labels_digest = _sha256_file(label_path)
if points_digest != RELLIS_EXAMPLE_POINTS_SHA256:
raise RellisSmokeError("RELLIS official example point digest is incompatible")
if labels_digest != RELLIS_EXAMPLE_LABELS_SHA256:
raise RellisSmokeError("RELLIS official example label digest is incompatible")
try:
frame = read_semantic_kitti_frame(point_path, label_path)
except DatasetFrameError as exc:
raise RellisSmokeError("RELLIS official example violates XYZI/label alignment") from exc
preview = rellis_native_scan_preview(
frame,
frame_id=RELLIS_EXAMPLE_FRAME_ID,
maximum_points=preview_points,
source_evidence={
"repository_url": RELLIS_REPOSITORY_URL,
"repository_commit": RELLIS_REPOSITORY_COMMIT,
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
"point_sha256": points_digest,
"label_sha256": labels_digest,
"license": RELLIS_LICENSE,
},
)
_atomic_json(output_path, preview)
return preview
def rellis_native_scan_preview(
frame: DatasetPointFrame,
*,
frame_id: str,
maximum_points: int,
source_evidence: dict[str, str],
) -> dict[str, Any]:
"""Build the common viewer artifact from a validated RELLIS native frame."""
if not frame_id or not 1 <= maximum_points <= MAX_RELLIS_PREVIEW_POINTS:
raise RellisSmokeError("RELLIS preview bounds are incompatible")
unknown = sorted(
int(value)
for value in np.unique(frame.semantic_labels)
if int(value) not in RELLIS_CLASS_BY_ID
)
if unknown:
raise RellisSmokeError("RELLIS frame contains labels absent from the pinned ontology")
required_evidence = {
"repository_url",
"repository_commit",
"label_config_sha256",
"point_sha256",
"label_sha256",
"license",
}
if set(source_evidence) != required_evidence or any(
not isinstance(value, str) or not value for value in source_evidence.values()
):
raise RellisSmokeError("RELLIS source evidence is incomplete")
sample_count = min(frame.point_count, maximum_points)
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
points = frame.points_xyz_m[indices]
remission = frame.remission[indices]
semantic = frame.semantic_labels[indices]
remission_min = float(np.min(remission))
remission_max = float(np.max(remission))
remission_span = remission_max - remission_min
if remission_span <= 0:
remission_u8 = np.zeros(sample_count, dtype=np.uint8)
else:
remission_u8 = np.rint(
(remission - remission_min) / remission_span * 255.0
).astype(np.uint8)
ground = np.zeros(sample_count, dtype=np.uint8)
evaluated = np.zeros(sample_count, dtype=np.uint8)
colors = np.zeros((sample_count, 3), dtype=np.uint8)
for label_id in np.unique(semantic):
item = RELLIS_CLASS_BY_ID[int(label_id)]
mask = semantic == label_id
colors[mask] = item.rgb
if item.ground_target != "ignore":
evaluated[mask] = 1
if item.ground_target == "ground":
ground[mask] = 1
present = Counter(int(value) for value in frame.semantic_labels)
present_classes = [
{
**RELLIS_CLASS_BY_ID[label_id].to_preview_dict(),
"source_point_count": count,
}
for label_id, count in sorted(present.items())
]
return {
"schema_version": RELLIS_PREVIEW_SCHEMA,
"source_id": RELLIS_SOURCE_ID,
"frame_id": frame_id,
"representation": "native-scan",
"sampling": "deterministic-even-index",
"source_point_count": frame.point_count,
"point_count": sample_count,
"points_xyz_m": points.tolist(),
"remission_0_to_255": remission_u8.tolist(),
"semantic_label_ids": semantic.astype(np.uint16).tolist(),
"semantic_rgb_0_to_255": colors.reshape(-1).tolist(),
"ground_truth_ground": ground.tolist(),
"evaluation_mask": evaluated.tolist(),
"classes": present_classes,
"coordinate_frame": {
"frame_id": "sensor/lidar/os1",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"transform_applied": False,
},
"ground_policy": {
"schema_version": RELLIS_GROUND_POLICY_SCHEMA,
"ground": [
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ground"
],
"non_ground": [
item.class_name
for item in RELLIS_CLASSES
if item.ground_target == "non-ground"
],
"ignore": [
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ignore"
],
},
"source_evidence": source_evidence,
"safety": {
"visualization_only": True,
"compatibility_smoke_only": True,
"navigation_or_safety_accepted": False,
},
}
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
try:
with path.open("rb") as source:
for chunk in iter(lambda: source.read(1024**2), b""):
digest.update(chunk)
except OSError as exc:
raise RellisSmokeError("RELLIS official example is unavailable") from exc
return digest.hexdigest()
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
mode="w",
dir=path.parent,
encoding="utf-8",
delete=False,
) as temporary:
temporary_path = Path(temporary.name)
json.dump(document, temporary, ensure_ascii=False, separators=(",", ":"))
temporary.flush()
os.fsync(temporary.fileno())
os.replace(temporary_path, path)

View File

@ -426,6 +426,9 @@ app.include_router(
dataset_preview_provider=lambda: ( dataset_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json" REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
), ),
dataset_rellis_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
),
dataset_ground_preview_provider=lambda: ( dataset_ground_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json" REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
), ),

View File

@ -4,7 +4,7 @@ import os
import re import re
from collections.abc import Callable from collections.abc import Callable
from pathlib import Path from pathlib import Path
from typing import Any, Final from typing import Annotated, Any, Final
from fastapi import APIRouter, HTTPException, Query, Response from fastapi import APIRouter, HTTPException, Query, Response
@ -21,6 +21,7 @@ from k1link.compute import (
lidar_pack_detail, lidar_pack_detail,
) )
from k1link.datasets import ( from k1link.datasets import (
RELLIS_SOURCE_ID,
DatasetAdmissionError, DatasetAdmissionError,
configured_dataset_admission_manifest, configured_dataset_admission_manifest,
configured_dataset_ground_preview, configured_dataset_ground_preview,
@ -62,6 +63,7 @@ def build_lidar_router(
field_review_root_provider: RootProvider = configured_lidar_field_review_root, field_review_root_provider: RootProvider = configured_lidar_field_review_root,
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest, dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview, dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview, dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
) -> APIRouter: ) -> APIRouter:
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"]) router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
@ -70,18 +72,32 @@ def build_lidar_router(
def get_dataset_gateway() -> dict[str, object]: def get_dataset_gateway() -> dict[str, object]:
return dataset_gateway_catalog( return dataset_gateway_catalog(
admission_manifest_path=dataset_admission_provider(), admission_manifest_path=dataset_admission_provider(),
rellis_preview_path=dataset_rellis_preview_provider(),
) )
@router.get("/dataset-gateway/preview") @router.get("/dataset-gateway/preview")
def get_dataset_gateway_preview() -> dict[str, Any]: def get_dataset_gateway_preview(
source_id: Annotated[
str,
Query(min_length=1, max_length=160),
] = "goose-3d/v2025-08-22",
) -> dict[str, Any]:
if source_id == "goose-3d/v2025-08-22":
path = dataset_preview_provider() path = dataset_preview_provider()
elif source_id == RELLIS_SOURCE_ID:
path = dataset_rellis_preview_provider()
else:
raise HTTPException(status_code=404, detail="Dataset source не найден.")
if path is None or not path.is_file(): if path is None or not path.is_file():
raise HTTPException( raise HTTPException(
status_code=404, status_code=404,
detail="Первый размеченный native scan ещё не импортирован.", detail="Размеченный native scan источника ещё не импортирован.",
) )
try: try:
return read_dataset_native_scan_preview(path) preview = read_dataset_native_scan_preview(path)
if preview["source_id"] != source_id:
raise DatasetAdmissionError("dataset preview source mismatch")
return preview
except DatasetAdmissionError as exc: except DatasetAdmissionError as exc:
raise HTTPException( raise HTTPException(
status_code=500, status_code=500,

View File

@ -12,6 +12,7 @@ from fastapi.routing import APIRoute
from k1link.datasets import ( from k1link.datasets import (
DatasetAdmissionError, DatasetAdmissionError,
DatasetFrameError, DatasetFrameError,
RELLIS_SOURCE_ID,
dataset_gateway_catalog, dataset_gateway_catalog,
goose_admission, goose_admission,
goose_benchmark, goose_benchmark,
@ -19,6 +20,7 @@ from k1link.datasets import (
read_dataset_ground_preview, read_dataset_ground_preview,
read_dataset_native_scan_preview, read_dataset_native_scan_preview,
read_semantic_kitti_frame, read_semantic_kitti_frame,
rellis_native_scan_preview,
) )
from k1link.datasets.goose_admission import admit_goose_validation from k1link.datasets.goose_admission import admit_goose_validation
from k1link.datasets.goose_benchmark import ( from k1link.datasets.goose_benchmark import (
@ -111,10 +113,14 @@ def test_dataset_gateway_api_is_read_only_and_path_free(
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway") route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
response = route() # type: ignore[operator] response = route() # type: ignore[operator]
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2" assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v3"
assert response["access"] == "read-only" assert response["access"] == "read-only"
assert response["storage"]["admitted"] is True assert response["storage"]["admitted"] is True
assert response["storage"]["path_exposed"] is False assert response["storage"]["path_exposed"] is False
assert [source["source_id"] for source in response["sources"]] == [
"goose-3d/v2025-08-22",
RELLIS_SOURCE_ID,
]
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace( assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
response["storage"]["required_wsl_root"], # type: ignore[index] response["storage"]["required_wsl_root"], # type: ignore[index]
"", "",
@ -286,6 +292,70 @@ def test_dataset_preview_api_is_read_only_and_integrity_checked(
assert response["frame_id"] == "frame-1" assert response["frame_id"] == "frame-1"
def test_rellis_smoke_preview_keeps_semantickitti_alignment_and_ignore_policy(
tmp_path: Path,
) -> None:
points_path = tmp_path / "000104.bin"
labels_path = tmp_path / "000104.label"
np.asarray(
[
[1.0, 0.0, -1.0, 0.1],
[2.0, 0.2, -0.8, 0.2],
[3.0, -0.2, 0.5, 0.3],
[4.0, 0.0, -0.5, 0.4],
],
dtype="<f4",
).tofile(points_path)
np.asarray([3, 23, 4, 31], dtype="<u4").tofile(labels_path)
frame = read_semantic_kitti_frame(points_path, labels_path)
preview = rellis_native_scan_preview(
frame,
frame_id="000104",
maximum_points=4,
source_evidence={
"repository_url": "https://github.com/unmannedlab/RELLIS-3D",
"repository_commit": "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
"label_config_sha256": (
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
),
"point_sha256": (
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
),
"label_sha256": (
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
),
"license": "CC-BY-NC-SA-3.0",
},
)
preview_path = tmp_path / "rellis-preview.json"
preview_path.write_text(json.dumps(preview), encoding="utf-8")
decoded = read_dataset_native_scan_preview(preview_path)
assert decoded["source_id"] == RELLIS_SOURCE_ID
assert decoded["ground_truth_ground"] == [1, 1, 0, 0]
assert decoded["evaluation_mask"] == [1, 1, 1, 0]
assert decoded["coordinate_frame"]["x"] == "forward"
assert decoded["safety"]["compatibility_smoke_only"] is True
router = build_lidar_router(
dataset_admission_provider=lambda: None,
dataset_preview_provider=lambda: None,
dataset_rellis_preview_provider=lambda: preview_path,
)
route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview")
response = route(source_id=RELLIS_SOURCE_ID) # type: ignore[operator]
assert response["frame_id"] == "000104"
catalog = dataset_gateway_catalog(
Path("/mnt/d/NDC_MISSIONCORE/datasets"),
rellis_preview_path=preview_path,
)
rellis = catalog["sources"][1] # type: ignore[index]
assert rellis["admission"]["status"] == "smoke-ready" # type: ignore[index]
def test_goose_current_ground_is_scored_against_independent_labels( def test_goose_current_ground_is_scored_against_independent_labels(
tmp_path: Path, tmp_path: Path,
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,