feat: add RELLIS dataset compatibility smoke
This commit is contained in:
parent
13c35ff446
commit
054feec0d2
|
|
@ -3,6 +3,57 @@ export type DatasetRepresentationId =
|
||||||
| "normalized-scan"
|
| "normalized-scan"
|
||||||
| "rolling-local-map";
|
| "rolling-local-map";
|
||||||
|
|
||||||
|
export type DatasetSourceAdmissionStatus =
|
||||||
|
| "blocked-storage-policy"
|
||||||
|
| "ready-for-download"
|
||||||
|
| "ready-for-smoke"
|
||||||
|
| "downloading"
|
||||||
|
| "downloaded"
|
||||||
|
| "verifying"
|
||||||
|
| "verified"
|
||||||
|
| "frame-ready"
|
||||||
|
| "smoke-ready";
|
||||||
|
|
||||||
|
export interface DatasetSource {
|
||||||
|
sourceId: string;
|
||||||
|
sourceKind: "goose" | "rellis";
|
||||||
|
displayName: string;
|
||||||
|
role: string;
|
||||||
|
license: string;
|
||||||
|
commercialUse:
|
||||||
|
| "allowed-with-share-alike"
|
||||||
|
| "research-only-license-review-required";
|
||||||
|
format: string;
|
||||||
|
frameSemantics: "one-lidar-revolution";
|
||||||
|
sensor: string;
|
||||||
|
platforms: string[];
|
||||||
|
superclasses: string[];
|
||||||
|
smokeExampleMb: number | null;
|
||||||
|
primaryScanArchiveGb: number;
|
||||||
|
labelArchiveGb: number | null;
|
||||||
|
posesArchiveGb: number | null;
|
||||||
|
fragmentCount: number;
|
||||||
|
annotatedScanCount: number;
|
||||||
|
admissionStatus: DatasetSourceAdmissionStatus;
|
||||||
|
archive: {
|
||||||
|
bytesTransferred: number;
|
||||||
|
totalBytes: number;
|
||||||
|
sizeBytes: number | null;
|
||||||
|
sha256: string | null;
|
||||||
|
integrity: string;
|
||||||
|
vendorChecksumAvailable: false;
|
||||||
|
} | null;
|
||||||
|
frame: {
|
||||||
|
frameId: string;
|
||||||
|
pointCount: number;
|
||||||
|
semanticClassCount: number;
|
||||||
|
groundTruthGroundFraction: number;
|
||||||
|
previewPointCount: number;
|
||||||
|
previewSha256: string | null;
|
||||||
|
previewAvailable: true;
|
||||||
|
} | null;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DatasetGatewayCatalog {
|
export interface DatasetGatewayCatalog {
|
||||||
storage: {
|
storage: {
|
||||||
configured: boolean;
|
configured: boolean;
|
||||||
|
|
@ -13,42 +64,7 @@ export interface DatasetGatewayCatalog {
|
||||||
requiredWindowsRoot: string;
|
requiredWindowsRoot: string;
|
||||||
requiredWslRoot: string;
|
requiredWslRoot: string;
|
||||||
};
|
};
|
||||||
source: {
|
sources: DatasetSource[];
|
||||||
sourceId: string;
|
|
||||||
displayName: string;
|
|
||||||
role: string;
|
|
||||||
license: string;
|
|
||||||
format: string;
|
|
||||||
frameSemantics: "one-lidar-revolution";
|
|
||||||
platforms: string[];
|
|
||||||
superclasses: string[];
|
|
||||||
validationArchiveGb: number;
|
|
||||||
admissionStatus:
|
|
||||||
| "blocked-storage-policy"
|
|
||||||
| "ready-for-download"
|
|
||||||
| "downloading"
|
|
||||||
| "downloaded"
|
|
||||||
| "verifying"
|
|
||||||
| "verified"
|
|
||||||
| "frame-ready";
|
|
||||||
archive: {
|
|
||||||
bytesTransferred: number;
|
|
||||||
totalBytes: number;
|
|
||||||
sizeBytes: number | null;
|
|
||||||
sha256: string | null;
|
|
||||||
integrity: string;
|
|
||||||
vendorChecksumAvailable: false;
|
|
||||||
} | null;
|
|
||||||
frame: {
|
|
||||||
frameId: string;
|
|
||||||
pointCount: number;
|
|
||||||
semanticClassCount: number;
|
|
||||||
groundTruthGroundFraction: number;
|
|
||||||
previewPointCount: number;
|
|
||||||
previewSha256: string;
|
|
||||||
previewAvailable: true;
|
|
||||||
} | null;
|
|
||||||
};
|
|
||||||
representations: Array<{
|
representations: Array<{
|
||||||
id: DatasetRepresentationId;
|
id: DatasetRepresentationId;
|
||||||
title: string;
|
title: string;
|
||||||
|
|
@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DatasetNativeScanPreview {
|
export interface DatasetNativeScanPreview {
|
||||||
sourceId: "goose-3d/v2025-08-22";
|
sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
|
||||||
frameId: string;
|
frameId: string;
|
||||||
sourcePointCount: number;
|
sourcePointCount: number;
|
||||||
pointCount: number;
|
pointCount: number;
|
||||||
|
|
@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
|
||||||
semanticLabelIds: number[];
|
semanticLabelIds: number[];
|
||||||
semanticRgb0To255: number[];
|
semanticRgb0To255: number[];
|
||||||
groundTruthGround: number[];
|
groundTruthGround: number[];
|
||||||
|
evaluationMask: number[];
|
||||||
classes: Array<{
|
classes: Array<{
|
||||||
labelId: number;
|
labelId: number;
|
||||||
className: string;
|
className: string;
|
||||||
hex: string;
|
hex: string;
|
||||||
challengeCategoryId: number;
|
challengeCategoryId: number | null;
|
||||||
challengeCategoryName: string;
|
challengeCategoryName: string | null;
|
||||||
|
groundTarget: "ground" | "non-ground" | "ignore" | null;
|
||||||
|
sourcePointCount: number | null;
|
||||||
}>;
|
}>;
|
||||||
|
coordinateFrame: {
|
||||||
|
frameId: string;
|
||||||
|
x: "forward";
|
||||||
|
y: "left";
|
||||||
|
z: "up";
|
||||||
|
} | null;
|
||||||
|
compatibilitySmokeOnly: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DatasetGroundMetrics {
|
export interface DatasetGroundMetrics {
|
||||||
|
|
@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
|
||||||
): DatasetGatewayCatalog {
|
): DatasetGatewayCatalog {
|
||||||
const source = record(value, "Dataset Gateway");
|
const source = record(value, "Dataset Gateway");
|
||||||
if (
|
if (
|
||||||
source.schema_version !== "missioncore.dataset-gateway-catalog/v2"
|
source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
|
||||||
|| source.access !== "read-only"
|
|| source.access !== "read-only"
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
|
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
|
||||||
|
|
@ -247,85 +273,158 @@ export function parseDatasetGatewayCatalog(
|
||||||
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
|
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
|
||||||
}
|
}
|
||||||
const sources = array(source.sources, "sources");
|
const sources = array(source.sources, "sources");
|
||||||
if (sources.length !== 1) {
|
if (sources.length !== 2) {
|
||||||
throw new DatasetGatewayContractError("Ожидался один первичный dataset source");
|
throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
|
||||||
}
|
}
|
||||||
const dataset = record(sources[0], "sources[0]");
|
const parsedSources = sources.map((value, index): DatasetSource => {
|
||||||
const download = record(dataset.download, "source.download");
|
const dataset = record(value, `sources[${index}]`);
|
||||||
const admission = record(dataset.admission, "source.admission");
|
const sourceKind = string(dataset.source_kind, "source_kind", true);
|
||||||
if (download.automatic !== false) {
|
const sourceId = string(dataset.source_id, "source_id", true);
|
||||||
throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически");
|
if (
|
||||||
}
|
(sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
|
||||||
const admissionStatus = admission.status;
|
&& (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("Dataset source identity неизвестна");
|
||||||
|
}
|
||||||
|
if (dataset.frame_semantics !== "one-lidar-revolution") {
|
||||||
|
throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
|
||||||
|
}
|
||||||
|
const download = record(dataset.download, `sources[${index}].download`);
|
||||||
|
const statistics = record(dataset.statistics, `sources[${index}].statistics`);
|
||||||
|
const admission = record(dataset.admission, `sources[${index}].admission`);
|
||||||
|
if (download.automatic !== false) {
|
||||||
|
throw new DatasetGatewayContractError(
|
||||||
|
"Большой dataset нельзя загружать автоматически",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const admissionStatus = admission.status;
|
||||||
|
if (
|
||||||
|
admissionStatus !== "ready-for-download"
|
||||||
|
&& admissionStatus !== "ready-for-smoke"
|
||||||
|
&& admissionStatus !== "blocked-storage-policy"
|
||||||
|
&& admissionStatus !== "downloading"
|
||||||
|
&& admissionStatus !== "downloaded"
|
||||||
|
&& admissionStatus !== "verifying"
|
||||||
|
&& admissionStatus !== "verified"
|
||||||
|
&& admissionStatus !== "frame-ready"
|
||||||
|
&& admissionStatus !== "smoke-ready"
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("source admission status неизвестен");
|
||||||
|
}
|
||||||
|
const archive = admission.archive === null
|
||||||
|
? null
|
||||||
|
: record(admission.archive, `sources[${index}].admission.archive`);
|
||||||
|
const archiveValue = archive
|
||||||
|
? {
|
||||||
|
bytesTransferred: number(
|
||||||
|
archive.bytes_transferred,
|
||||||
|
"archive.bytes_transferred",
|
||||||
|
),
|
||||||
|
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
|
||||||
|
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
|
||||||
|
sha256: nullableDigest(archive.sha256, "archive.sha256"),
|
||||||
|
integrity: string(archive.integrity, "archive.integrity", true),
|
||||||
|
vendorChecksumAvailable: (() => {
|
||||||
|
if (archive.vendor_checksum_available !== false) {
|
||||||
|
throw new DatasetGatewayContractError(
|
||||||
|
"archive.vendor_checksum_available: ожидался false",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return false as const;
|
||||||
|
})(),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
const admittedFrame = admission.frame === null
|
||||||
|
? null
|
||||||
|
: record(admission.frame, `sources[${index}].admission.frame`);
|
||||||
|
const frameValue = admittedFrame
|
||||||
|
? {
|
||||||
|
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
|
||||||
|
pointCount: number(admittedFrame.point_count, "frame.point_count"),
|
||||||
|
semanticClassCount: number(
|
||||||
|
admittedFrame.semantic_class_count,
|
||||||
|
"frame.semantic_class_count",
|
||||||
|
),
|
||||||
|
groundTruthGroundFraction: number(
|
||||||
|
admittedFrame.ground_truth_ground_fraction,
|
||||||
|
"frame.ground_truth_ground_fraction",
|
||||||
|
),
|
||||||
|
previewPointCount: number(
|
||||||
|
admittedFrame.preview_point_count,
|
||||||
|
"frame.preview_point_count",
|
||||||
|
),
|
||||||
|
previewSha256: nullableDigest(
|
||||||
|
admittedFrame.preview_sha256,
|
||||||
|
"frame.preview_sha256",
|
||||||
|
),
|
||||||
|
previewAvailable: (() => {
|
||||||
|
if (admittedFrame.preview_available !== true) {
|
||||||
|
throw new DatasetGatewayContractError(
|
||||||
|
"frame.preview_available: ожидался true",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return true as const;
|
||||||
|
})(),
|
||||||
|
}
|
||||||
|
: null;
|
||||||
|
if (
|
||||||
|
sourceKind === "goose"
|
||||||
|
&& frameValue
|
||||||
|
&& frameValue.previewSha256 === null
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
|
||||||
|
}
|
||||||
|
const commercialUse = dataset.commercial_use;
|
||||||
|
if (
|
||||||
|
commercialUse !== "allowed-with-share-alike"
|
||||||
|
&& commercialUse !== "research-only-license-review-required"
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("Dataset license scope неизвестен");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
sourceId,
|
||||||
|
sourceKind,
|
||||||
|
displayName: string(dataset.display_name, "display_name"),
|
||||||
|
role: string(dataset.role, "role", true),
|
||||||
|
license: string(dataset.license, "license"),
|
||||||
|
commercialUse,
|
||||||
|
format: string(dataset.format, "format", true),
|
||||||
|
frameSemantics: "one-lidar-revolution",
|
||||||
|
sensor: string(dataset.sensor, "sensor"),
|
||||||
|
platforms: displayStrings(dataset.platforms, "platforms"),
|
||||||
|
superclasses: strings(dataset.superclasses, "superclasses"),
|
||||||
|
smokeExampleMb: nullableNumber(
|
||||||
|
download.smoke_example_mb,
|
||||||
|
"smoke_example_mb",
|
||||||
|
),
|
||||||
|
primaryScanArchiveGb: number(
|
||||||
|
download.primary_scan_archive_gb,
|
||||||
|
"primary_scan_archive_gb",
|
||||||
|
),
|
||||||
|
labelArchiveGb: nullableNumber(
|
||||||
|
download.label_archive_gb,
|
||||||
|
"label_archive_gb",
|
||||||
|
),
|
||||||
|
posesArchiveGb: nullableNumber(
|
||||||
|
download.poses_archive_gb,
|
||||||
|
"poses_archive_gb",
|
||||||
|
),
|
||||||
|
fragmentCount: number(statistics.fragment_count, "fragment_count"),
|
||||||
|
annotatedScanCount: number(
|
||||||
|
statistics.annotated_scan_count,
|
||||||
|
"annotated_scan_count",
|
||||||
|
),
|
||||||
|
admissionStatus,
|
||||||
|
archive: archiveValue,
|
||||||
|
frame: frameValue,
|
||||||
|
};
|
||||||
|
});
|
||||||
if (
|
if (
|
||||||
admissionStatus !== "ready-for-download"
|
parsedSources[0]?.sourceKind !== "goose"
|
||||||
&& admissionStatus !== "blocked-storage-policy"
|
|| parsedSources[1]?.sourceKind !== "rellis"
|
||||||
&& admissionStatus !== "downloading"
|
|
||||||
&& admissionStatus !== "downloaded"
|
|
||||||
&& admissionStatus !== "verifying"
|
|
||||||
&& admissionStatus !== "verified"
|
|
||||||
&& admissionStatus !== "frame-ready"
|
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("source admission status неизвестен");
|
throw new DatasetGatewayContractError("Dataset source order несовместим");
|
||||||
}
|
}
|
||||||
const archive = admission.archive === null
|
|
||||||
? null
|
|
||||||
: record(admission.archive, "source.admission.archive");
|
|
||||||
const archiveValue = archive
|
|
||||||
? {
|
|
||||||
bytesTransferred: number(
|
|
||||||
archive.bytes_transferred,
|
|
||||||
"archive.bytes_transferred",
|
|
||||||
),
|
|
||||||
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
|
|
||||||
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
|
|
||||||
sha256: nullableDigest(archive.sha256, "archive.sha256"),
|
|
||||||
integrity: string(archive.integrity, "archive.integrity", true),
|
|
||||||
vendorChecksumAvailable: (() => {
|
|
||||||
if (archive.vendor_checksum_available !== false) {
|
|
||||||
throw new DatasetGatewayContractError(
|
|
||||||
"archive.vendor_checksum_available: ожидался false",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return false as const;
|
|
||||||
})(),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
const admittedFrame = admission.frame === null
|
|
||||||
? null
|
|
||||||
: record(admission.frame, "source.admission.frame");
|
|
||||||
const frameValue = admittedFrame
|
|
||||||
? {
|
|
||||||
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
|
|
||||||
pointCount: number(admittedFrame.point_count, "frame.point_count"),
|
|
||||||
semanticClassCount: number(
|
|
||||||
admittedFrame.semantic_class_count,
|
|
||||||
"frame.semantic_class_count",
|
|
||||||
),
|
|
||||||
groundTruthGroundFraction: number(
|
|
||||||
admittedFrame.ground_truth_ground_fraction,
|
|
||||||
"frame.ground_truth_ground_fraction",
|
|
||||||
),
|
|
||||||
previewPointCount: number(
|
|
||||||
admittedFrame.preview_point_count,
|
|
||||||
"frame.preview_point_count",
|
|
||||||
),
|
|
||||||
previewSha256: nullableDigest(
|
|
||||||
admittedFrame.preview_sha256,
|
|
||||||
"frame.preview_sha256",
|
|
||||||
) ?? (() => {
|
|
||||||
throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
|
|
||||||
})(),
|
|
||||||
previewAvailable: (() => {
|
|
||||||
if (admittedFrame.preview_available !== true) {
|
|
||||||
throw new DatasetGatewayContractError(
|
|
||||||
"frame.preview_available: ожидался true",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return true as const;
|
|
||||||
})(),
|
|
||||||
}
|
|
||||||
: null;
|
|
||||||
const representations = array(
|
const representations = array(
|
||||||
source.representations,
|
source.representations,
|
||||||
"representations",
|
"representations",
|
||||||
|
|
@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Vendor-map boundary завышен");
|
throw new DatasetGatewayContractError("Vendor-map boundary завышен");
|
||||||
}
|
}
|
||||||
if (dataset.frame_semantics !== "one-lidar-revolution") {
|
|
||||||
throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
|
|
||||||
}
|
|
||||||
return {
|
return {
|
||||||
storage: {
|
storage: {
|
||||||
configured: boolean(storage.configured, "storage.configured"),
|
configured: boolean(storage.configured, "storage.configured"),
|
||||||
|
|
@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
|
||||||
),
|
),
|
||||||
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
|
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
|
||||||
},
|
},
|
||||||
source: {
|
sources: parsedSources,
|
||||||
sourceId: string(dataset.source_id, "source_id", true),
|
|
||||||
displayName: string(dataset.display_name, "display_name"),
|
|
||||||
role: string(dataset.role, "role", true),
|
|
||||||
license: string(dataset.license, "license"),
|
|
||||||
format: string(dataset.format, "format", true),
|
|
||||||
frameSemantics: "one-lidar-revolution",
|
|
||||||
platforms: displayStrings(dataset.platforms, "platforms"),
|
|
||||||
superclasses: strings(dataset.superclasses, "superclasses"),
|
|
||||||
validationArchiveGb: number(
|
|
||||||
download.validation_archive_gb,
|
|
||||||
"validation_archive_gb",
|
|
||||||
),
|
|
||||||
admissionStatus,
|
|
||||||
archive: archiveValue,
|
|
||||||
frame: frameValue,
|
|
||||||
},
|
|
||||||
representations,
|
representations,
|
||||||
pipeline,
|
pipeline,
|
||||||
currentInput: {
|
currentInput: {
|
||||||
|
|
@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
|
||||||
value: unknown,
|
value: unknown,
|
||||||
): DatasetNativeScanPreview {
|
): DatasetNativeScanPreview {
|
||||||
const source = record(value, "Dataset preview");
|
const source = record(value, "Dataset preview");
|
||||||
|
const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
|
||||||
|
&& source.source_id === "goose-3d/v2025-08-22";
|
||||||
|
const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
|
||||||
|
&& source.source_id === "rellis-3d/v1.1";
|
||||||
if (
|
if (
|
||||||
source.schema_version !== "missioncore.dataset-native-scan-preview/v1"
|
(!isGoose && !isRellis)
|
||||||
|| source.source_id !== "goose-3d/v2025-08-22"
|
|
||||||
|| source.representation !== "native-scan"
|
|| source.representation !== "native-scan"
|
||||||
|| source.sampling !== "deterministic-even-index"
|
|| source.sampling !== "deterministic-even-index"
|
||||||
) {
|
) {
|
||||||
|
|
@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
|
||||||
"ground_truth_ground",
|
"ground_truth_ground",
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
|
const evaluationMask = isRellis
|
||||||
|
? integers(source.evaluation_mask, "evaluation_mask", 1)
|
||||||
|
: Array.from({ length: pointCount }, () => 1);
|
||||||
if (
|
if (
|
||||||
pointCount < 1
|
pointCount < 1
|
||||||
|| pointCount > 50_000
|
|| pointCount > 50_000
|
||||||
|
|
@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
|
||||||
|| semanticLabelIds.length !== pointCount
|
|| semanticLabelIds.length !== pointCount
|
||||||
|| semanticRgb0To255.length !== pointCount * 3
|
|| semanticRgb0To255.length !== pointCount * 3
|
||||||
|| groundTruthGround.length !== pointCount
|
|| groundTruthGround.length !== pointCount
|
||||||
|
|| evaluationMask.length !== pointCount
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
|
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
|
||||||
}
|
}
|
||||||
const classes = array(source.classes, "classes").map((value, index) => {
|
const classes = array(source.classes, "classes").map((value, index) => {
|
||||||
const item = record(value, `classes[${index}]`);
|
const item = record(value, `classes[${index}]`);
|
||||||
|
const groundTarget = item.ground_target;
|
||||||
|
if (
|
||||||
|
groundTarget !== undefined
|
||||||
|
&& groundTarget !== "ground"
|
||||||
|
&& groundTarget !== "non-ground"
|
||||||
|
&& groundTarget !== "ignore"
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("class.ground_target неизвестен");
|
||||||
|
}
|
||||||
|
const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
|
||||||
|
groundTarget === undefined ? null : groundTarget;
|
||||||
return {
|
return {
|
||||||
labelId: number(item.label_id, "class.label_id"),
|
labelId: number(item.label_id, "class.label_id"),
|
||||||
className: string(item.class_name, "class.class_name", true),
|
className: string(item.class_name, "class.class_name", true),
|
||||||
hex: string(item.hex, "class.hex"),
|
hex: string(item.hex, "class.hex"),
|
||||||
challengeCategoryId: number(
|
challengeCategoryId: item.challenge_category_id === undefined
|
||||||
item.challenge_category_id,
|
? null
|
||||||
"class.challenge_category_id",
|
: number(item.challenge_category_id, "class.challenge_category_id"),
|
||||||
),
|
challengeCategoryName: item.challenge_category_name === undefined
|
||||||
challengeCategoryName: string(
|
? null
|
||||||
item.challenge_category_name,
|
: string(
|
||||||
"class.challenge_category_name",
|
item.challenge_category_name,
|
||||||
true,
|
"class.challenge_category_name",
|
||||||
),
|
true,
|
||||||
|
),
|
||||||
|
groundTarget: normalizedGroundTarget,
|
||||||
|
sourcePointCount: item.source_point_count === undefined
|
||||||
|
? null
|
||||||
|
: number(item.source_point_count, "class.source_point_count"),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
const safety = record(source.safety, "safety");
|
const safety = record(source.safety, "safety");
|
||||||
if (
|
if (
|
||||||
safety.visualization_only !== true
|
safety.visualization_only !== true
|
||||||
|| safety.navigation_or_safety_accepted !== false
|
|| safety.navigation_or_safety_accepted !== false
|
||||||
|
|| (isRellis && safety.compatibility_smoke_only !== true)
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
|
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
|
||||||
}
|
}
|
||||||
|
const coordinateFrame = isRellis
|
||||||
|
? (() => {
|
||||||
|
const frame = record(source.coordinate_frame, "coordinate_frame");
|
||||||
|
if (
|
||||||
|
frame.frame_id !== "sensor/lidar/os1"
|
||||||
|
|| frame.x !== "forward"
|
||||||
|
|| frame.y !== "left"
|
||||||
|
|| frame.z !== "up"
|
||||||
|
|| frame.handedness !== "right"
|
||||||
|
|| frame.transform_applied !== false
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
frameId: "sensor/lidar/os1",
|
||||||
|
x: "forward" as const,
|
||||||
|
y: "left" as const,
|
||||||
|
z: "up" as const,
|
||||||
|
};
|
||||||
|
})()
|
||||||
|
: null;
|
||||||
return {
|
return {
|
||||||
sourceId: "goose-3d/v2025-08-22",
|
sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
|
||||||
frameId: string(source.frame_id, "frame_id", true),
|
frameId: string(source.frame_id, "frame_id", true),
|
||||||
sourcePointCount,
|
sourcePointCount,
|
||||||
pointCount,
|
pointCount,
|
||||||
|
|
@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
|
||||||
semanticLabelIds,
|
semanticLabelIds,
|
||||||
semanticRgb0To255,
|
semanticRgb0To255,
|
||||||
groundTruthGround,
|
groundTruthGround,
|
||||||
|
evaluationMask,
|
||||||
classes,
|
classes,
|
||||||
|
coordinateFrame,
|
||||||
|
compatibilitySmokeOnly: isRellis,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchDatasetNativeScanPreview(
|
export async function fetchDatasetNativeScanPreview(
|
||||||
options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {},
|
options: {
|
||||||
|
sourceId?: DatasetNativeScanPreview["sourceId"];
|
||||||
|
signal?: AbortSignal;
|
||||||
|
fetcher?: DatasetFetch;
|
||||||
|
} = {},
|
||||||
): Promise<DatasetNativeScanPreview> {
|
): Promise<DatasetNativeScanPreview> {
|
||||||
const fetcher = options.fetcher ?? fetch;
|
const fetcher = options.fetcher ?? fetch;
|
||||||
const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", {
|
const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
|
||||||
|
const response = await fetcher(
|
||||||
|
`/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
|
||||||
|
{
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers: { Accept: "application/json" },
|
headers: { Accept: "application/json" },
|
||||||
signal: options.signal,
|
signal: options.signal,
|
||||||
});
|
},
|
||||||
|
);
|
||||||
return parseDatasetNativeScanPreview(await responseJson(response));
|
return parseDatasetNativeScanPreview(await responseJson(response));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -224,7 +224,9 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataset-fragment-comparison,
|
.dataset-fragment-comparison,
|
||||||
.dataset-fragment-risks {
|
.dataset-fragment-risks,
|
||||||
|
.rellis-review__metrics,
|
||||||
|
.rellis-review__policy > div {
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1223,6 +1223,85 @@
|
||||||
line-height: 1.5;
|
line-height: 1.5;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.rellis-review__metrics {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 0.35rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__metrics article {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.2rem;
|
||||||
|
border-radius: 0.85rem;
|
||||||
|
background: var(--station-panel);
|
||||||
|
padding: 0.78rem 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__metrics span,
|
||||||
|
.rellis-review__metrics small {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.55rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__metrics strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 1.05rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.7rem;
|
||||||
|
border-radius: 1rem;
|
||||||
|
background: var(--station-panel);
|
||||||
|
padding: 0.9rem 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy h3,
|
||||||
|
.rellis-review__policy p {
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy h3 {
|
||||||
|
margin-top: 0.28rem;
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy p {
|
||||||
|
max-width: 52rem;
|
||||||
|
margin-top: 0.3rem;
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.6rem;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy > div {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 0.35rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy article {
|
||||||
|
display: grid;
|
||||||
|
align-content: start;
|
||||||
|
gap: 0.35rem;
|
||||||
|
border-radius: 0.75rem;
|
||||||
|
background: rgb(255 255 255 / 0.03);
|
||||||
|
padding: 0.7rem 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy article span {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.54rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.rellis-review__policy article strong {
|
||||||
|
color: var(--nodedc-text-secondary);
|
||||||
|
font-size: 0.62rem;
|
||||||
|
font-weight: 560;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
|
||||||
.dataset-sequence-timeline {
|
.dataset-sequence-timeline {
|
||||||
height: 2rem;
|
height: 2rem;
|
||||||
}
|
}
|
||||||
|
|
@ -2043,6 +2122,11 @@
|
||||||
font-size: 1rem;
|
font-size: 1rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dataset-library__entries {
|
||||||
|
display: grid;
|
||||||
|
gap: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
.dataset-entry {
|
.dataset-entry {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);
|
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);
|
||||||
|
|
|
||||||
|
|
@ -8,12 +8,15 @@ import {
|
||||||
import {
|
import {
|
||||||
fetchDatasetGatewayCatalog,
|
fetchDatasetGatewayCatalog,
|
||||||
type DatasetGatewayCatalog,
|
type DatasetGatewayCatalog,
|
||||||
|
type DatasetSource,
|
||||||
|
type DatasetSourceAdmissionStatus,
|
||||||
} from "../core/lidar/datasetGateway";
|
} from "../core/lidar/datasetGateway";
|
||||||
import {
|
import {
|
||||||
fetchPolygonRunCatalog,
|
fetchPolygonRunCatalog,
|
||||||
type PolygonRunCatalog,
|
type PolygonRunCatalog,
|
||||||
} from "../core/polygon/runArchive";
|
} from "../core/polygon/runArchive";
|
||||||
import { GooseDatasetReview } from "./GooseDatasetReview";
|
import { GooseDatasetReview } from "./GooseDatasetReview";
|
||||||
|
import { RellisDatasetReview } from "./RellisDatasetReview";
|
||||||
|
|
||||||
const representationLabels: Record<string, string> = {
|
const representationLabels: Record<string, string> = {
|
||||||
"native-scan": "Исходный скан",
|
"native-scan": "Исходный скан",
|
||||||
|
|
@ -27,15 +30,17 @@ function errorMessage(error: unknown): string {
|
||||||
: "Каталог датасетов недоступен.";
|
: "Каталог датасетов недоступен.";
|
||||||
}
|
}
|
||||||
|
|
||||||
function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) {
|
function admissionLabel(status: DatasetSourceAdmissionStatus): string {
|
||||||
const labels: Record<typeof status, string> = {
|
const labels: Record<DatasetSourceAdmissionStatus, string> = {
|
||||||
"blocked-storage-policy": "Worker не настроен",
|
"blocked-storage-policy": "Worker не настроен",
|
||||||
"ready-for-download": "Готов к загрузке",
|
"ready-for-download": "Готов к загрузке",
|
||||||
|
"ready-for-smoke": "Готов к smoke",
|
||||||
downloading: "Загружается",
|
downloading: "Загружается",
|
||||||
downloaded: "Загружен",
|
downloaded: "Загружен",
|
||||||
verifying: "Проверяется",
|
verifying: "Проверяется",
|
||||||
verified: "Проверен",
|
verified: "Проверен",
|
||||||
"frame-ready": "Первый кадр готов",
|
"frame-ready": "Dataset готов",
|
||||||
|
"smoke-ready": "Smoke готов",
|
||||||
};
|
};
|
||||||
return labels[status];
|
return labels[status];
|
||||||
}
|
}
|
||||||
|
|
@ -55,12 +60,42 @@ function isGooseQualificationRun(
|
||||||
&& item.scenarioGeneration.startsWith("goose-3d");
|
&& item.scenarioGeneration.startsWith("goose-3d");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sourceReady(source: DatasetSource): boolean {
|
||||||
|
return source.admissionStatus === "frame-ready"
|
||||||
|
|| source.admissionStatus === "smoke-ready";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sourceDescription(source: DatasetSource): string {
|
||||||
|
if (source.sourceKind === "goose") {
|
||||||
|
return "Восемь фрагментов и 961 разреженный размеченный оборот VLS-128. Здесь уже опубликовано покадровое сравнение текущего ground baseline и Patchwork++.";
|
||||||
|
}
|
||||||
|
return "Независимый off-road источник с Ouster OS1 64. Сейчас открыт официальный frame 000104 для проверки reader, осей, ontology и ground-policy; полный benchmark ещё не выполнялся.";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sourceFooter(source: DatasetSource, storageReady: boolean): string {
|
||||||
|
if (source.admissionStatus === "frame-ready") {
|
||||||
|
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
|
||||||
|
}
|
||||||
|
if (source.admissionStatus === "smoke-ready") {
|
||||||
|
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
|
||||||
|
}
|
||||||
|
if (source.admissionStatus === "downloading") {
|
||||||
|
return "Архив остаётся на D worker; после загрузки проверяются digest, лицензия и point-label alignment.";
|
||||||
|
}
|
||||||
|
if (!storageReady) {
|
||||||
|
return "Сначала нужен допущенный Dataset Root на диске D worker.";
|
||||||
|
}
|
||||||
|
return source.sourceKind === "rellis"
|
||||||
|
? "Следующий шаг: пропустить официальный 24 МБ example через smoke admission."
|
||||||
|
: "Следующий шаг: загрузить validation archive на worker и проверить hash/license.";
|
||||||
|
}
|
||||||
|
|
||||||
export function DatasetGatewayWorkspace() {
|
export function DatasetGatewayWorkspace() {
|
||||||
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
|
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [reloadGeneration, setReloadGeneration] = useState(0);
|
const [reloadGeneration, setReloadGeneration] = useState(0);
|
||||||
const [previewOpen, setPreviewOpen] = useState(false);
|
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
|
||||||
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
|
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
|
||||||
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
|
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
|
||||||
const [runError, setRunError] = useState<string | null>(null);
|
const [runError, setRunError] = useState<string | null>(null);
|
||||||
|
|
@ -85,7 +120,9 @@ export function DatasetGatewayWorkspace() {
|
||||||
}, [reloadGeneration]);
|
}, [reloadGeneration]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (catalog?.source.admissionStatus !== "downloading") return;
|
if (!catalog?.sources.some((source) => source.admissionStatus === "downloading")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const timer = window.setTimeout(
|
const timer = window.setTimeout(
|
||||||
() => setReloadGeneration((value) => value + 1),
|
() => setReloadGeneration((value) => value + 1),
|
||||||
5_000,
|
5_000,
|
||||||
|
|
@ -113,10 +150,10 @@ export function DatasetGatewayWorkspace() {
|
||||||
return () => controller.abort();
|
return () => controller.abort();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const downloadProgress = catalog?.source.archive
|
|
||||||
? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes
|
|
||||||
: null;
|
|
||||||
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
|
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
|
||||||
|
const openSource = catalog?.sources.find(
|
||||||
|
(source) => source.sourceId === openSourceId,
|
||||||
|
) ?? null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="standard-workspace dataset-workspace">
|
<div className="standard-workspace dataset-workspace">
|
||||||
|
|
@ -125,17 +162,17 @@ export function DatasetGatewayWorkspace() {
|
||||||
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
|
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
|
||||||
<h2>Независимая проверка алгоритмов</h2>
|
<h2>Независимая проверка алгоритмов</h2>
|
||||||
<p>
|
<p>
|
||||||
Датасет содержит известные размеченные данные. Мы показываем
|
Каждый источник остаётся отдельным: собственные sensor domain,
|
||||||
конкретный фрагмент, результаты каждого алгоритма и ошибки
|
разметка, лицензия и результаты. Совпадение результата на GOOSE и
|
||||||
относительно разметки. Он не подмешивается в диагностику реального
|
RELLIS снижает риск подгонки под один датасет, но не заменяет replay
|
||||||
сенсора и не заменяет closed-loop симуляцию.
|
реального сенсора и closed-loop симуляцию.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<ol aria-label="Как использовать датасет">
|
<ol aria-label="Как использовать датасет">
|
||||||
<li><span>01</span>Выбрать фрагмент</li>
|
<li><span>01</span>Выбрать источник</li>
|
||||||
<li><span>02</span>Посмотреть сканы</li>
|
<li><span>02</span>Проверить scan и labels</li>
|
||||||
<li><span>03</span>Сравнить алгоритмы</li>
|
<li><span>03</span>Сравнить алгоритмы</li>
|
||||||
<li><span>04</span>Проверить риски</li>
|
<li><span>04</span>Проверить переносимость</li>
|
||||||
</ol>
|
</ol>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
|
@ -171,80 +208,91 @@ export function DatasetGatewayWorkspace() {
|
||||||
</StatusBadge>
|
</StatusBadge>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<article className="dataset-entry">
|
<div className="dataset-library__entries">
|
||||||
<div className="dataset-entry__identity">
|
{catalog.sources.map((source) => {
|
||||||
<span className="dataset-entry__mark">G</span>
|
const downloadProgress = source.archive
|
||||||
<div>
|
? source.archive.bytesTransferred / source.archive.totalBytes
|
||||||
<span>PUBLIC · OFF-ROAD · LABELED</span>
|
: null;
|
||||||
<h3>{catalog.source.displayName}</h3>
|
const opened = source.sourceId === openSourceId;
|
||||||
<p>
|
return (
|
||||||
Validation split: восемь фрагментов исходных записей и 961
|
<article
|
||||||
разреженный размеченный оборот VLS-128. Это коллекция native
|
className="dataset-entry"
|
||||||
scans для perception, а не непрерывное видео или траектория
|
data-source={source.sourceKind}
|
||||||
машины.
|
key={source.sourceId}
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<dl>
|
|
||||||
<div>
|
|
||||||
<dt>Состояние</dt>
|
|
||||||
<dd>{admissionLabel(catalog.source.admissionStatus)}</dd>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<dt>Validation</dt>
|
|
||||||
<dd>{catalog.source.validationArchiveGb} ГБ</dd>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<dt>Группы</dt>
|
|
||||||
<dd>{catalog.source.superclasses.length} superclass</dd>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<dt>Лицензия</dt>
|
|
||||||
<dd>{catalog.source.license}</dd>
|
|
||||||
</div>
|
|
||||||
</dl>
|
|
||||||
{catalog.source.admissionStatus === "downloading"
|
|
||||||
&& catalog.source.archive
|
|
||||||
&& downloadProgress !== null ? (
|
|
||||||
<div className="dataset-download" aria-label="Загрузка GOOSE validation">
|
|
||||||
<span style={{ width: `${downloadProgress * 100}%` }} />
|
|
||||||
<p>
|
|
||||||
{formatBytes(catalog.source.archive.bytesTransferred)}
|
|
||||||
{" из "}
|
|
||||||
{formatBytes(catalog.source.archive.totalBytes)}
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
) : null}
|
|
||||||
<footer>
|
|
||||||
<p>
|
|
||||||
{catalog.source.admissionStatus === "frame-ready"
|
|
||||||
? "Native scan и разметка прошли admission. Можно открыть независимый ground truth."
|
|
||||||
: catalog.source.admissionStatus === "downloading"
|
|
||||||
? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment."
|
|
||||||
: catalog.storage.admitted
|
|
||||||
? "Следующий шаг: загрузить validation archive на worker и проверить hash/license."
|
|
||||||
: "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}
|
|
||||||
</p>
|
|
||||||
<div>
|
|
||||||
<Button
|
|
||||||
size="compact"
|
|
||||||
variant="secondary"
|
|
||||||
disabled={catalog.source.admissionStatus !== "frame-ready"}
|
|
||||||
title={
|
|
||||||
catalog.source.admissionStatus === "frame-ready"
|
|
||||||
? "Открыть фрагменты, сканы и результаты анализа"
|
|
||||||
: "Размеченные сканы ещё не импортированы"
|
|
||||||
}
|
|
||||||
onClick={() => setPreviewOpen((value) => !value)}
|
|
||||||
>
|
>
|
||||||
{previewOpen ? "Закрыть датасет" : "Открыть датасет"}
|
<div className="dataset-entry__identity">
|
||||||
</Button>
|
<span className="dataset-entry__mark">
|
||||||
</div>
|
{source.sourceKind === "goose" ? "G" : "R"}
|
||||||
</footer>
|
</span>
|
||||||
</article>
|
<div>
|
||||||
|
<span>
|
||||||
|
PUBLIC · OFF-ROAD · LABELED
|
||||||
|
{source.sourceKind === "rellis" ? " · RESEARCH" : ""}
|
||||||
|
</span>
|
||||||
|
<h3>{source.displayName}</h3>
|
||||||
|
<p>{sourceDescription(source)}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<dl>
|
||||||
|
<div>
|
||||||
|
<dt>Состояние</dt>
|
||||||
|
<dd>{admissionLabel(source.admissionStatus)}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Сканер</dt>
|
||||||
|
<dd>{source.sensor}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Размеченных сканов</dt>
|
||||||
|
<dd>{source.annotatedScanCount.toLocaleString("ru-RU")}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Лицензия</dt>
|
||||||
|
<dd>{source.license}</dd>
|
||||||
|
</div>
|
||||||
|
</dl>
|
||||||
|
{source.admissionStatus === "downloading"
|
||||||
|
&& source.archive
|
||||||
|
&& downloadProgress !== null ? (
|
||||||
|
<div
|
||||||
|
className="dataset-download"
|
||||||
|
aria-label={`Загрузка ${source.displayName}`}
|
||||||
|
>
|
||||||
|
<span style={{ width: `${downloadProgress * 100}%` }} />
|
||||||
|
<p>
|
||||||
|
{formatBytes(source.archive.bytesTransferred)}
|
||||||
|
{" из "}
|
||||||
|
{formatBytes(source.archive.totalBytes)}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
<footer>
|
||||||
|
<p>{sourceFooter(source, catalog.storage.admitted)}</p>
|
||||||
|
<div>
|
||||||
|
<Button
|
||||||
|
size="compact"
|
||||||
|
variant="secondary"
|
||||||
|
disabled={!sourceReady(source)}
|
||||||
|
title={
|
||||||
|
sourceReady(source)
|
||||||
|
? "Открыть содержимое источника"
|
||||||
|
: "Совместимый preview ещё не опубликован"
|
||||||
|
}
|
||||||
|
onClick={() => setOpenSourceId(
|
||||||
|
opened ? null : source.sourceId,
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{opened ? "Закрыть датасет" : "Открыть датасет"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
|
</article>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
|
|
||||||
{previewOpen && selectedRunId ? (
|
{openSource?.sourceKind === "goose" && selectedRunId ? (
|
||||||
<>
|
<>
|
||||||
{gooseRuns.length > 1 ? (
|
{gooseRuns.length > 1 ? (
|
||||||
<div className="dataset-result-select">
|
<div className="dataset-result-select">
|
||||||
|
|
@ -267,12 +315,14 @@ export function DatasetGatewayWorkspace() {
|
||||||
) : null}
|
) : null}
|
||||||
<GooseDatasetReview runId={selectedRunId} />
|
<GooseDatasetReview runId={selectedRunId} />
|
||||||
</>
|
</>
|
||||||
) : previewOpen ? (
|
) : openSource?.sourceKind === "goose" ? (
|
||||||
<section className="polygon-review-unavailable">
|
<section className="polygon-review-unavailable">
|
||||||
<StatusBadge tone="warning">Review недоступен</StatusBadge>
|
<StatusBadge tone="warning">Review недоступен</StatusBadge>
|
||||||
<h3>Датасет принят, но результаты анализа не опубликованы</h3>
|
<h3>GOOSE принят, но анализ не опубликован</h3>
|
||||||
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
|
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
|
||||||
</section>
|
</section>
|
||||||
|
) : openSource?.sourceKind === "rellis" ? (
|
||||||
|
<RellisDatasetReview />
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<details className="dataset-contract">
|
<details className="dataset-contract">
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ export interface LidarGroundPointCloudFrame {
|
||||||
disagreement: number[];
|
disagreement: number[];
|
||||||
candidateDisagreement?: number[];
|
candidateDisagreement?: number[];
|
||||||
groundTruthGround?: number[];
|
groundTruthGround?: number[];
|
||||||
|
evaluationMask?: number[];
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -87,13 +88,18 @@ function frameColors(
|
||||||
);
|
);
|
||||||
} else if (mode === "ground-truth") {
|
} else if (mode === "ground-truth") {
|
||||||
const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
|
const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
|
||||||
setRgb(
|
const evaluated = frame.masks.evaluationMask?.[index] !== 0;
|
||||||
colors,
|
if (!evaluated) {
|
||||||
offset,
|
setRgb(colors, offset, 0.86, 0.61, 0.24);
|
||||||
groundTruth ? 0.76 : 0.29,
|
} else {
|
||||||
groundTruth ? 0.86 : 0.33,
|
setRgb(
|
||||||
groundTruth ? 0.52 : 0.36,
|
colors,
|
||||||
);
|
offset,
|
||||||
|
groundTruth ? 0.76 : 0.29,
|
||||||
|
groundTruth ? 0.86 : 0.33,
|
||||||
|
groundTruth ? 0.52 : 0.36,
|
||||||
|
);
|
||||||
|
}
|
||||||
} else if (mode === "current") {
|
} else if (mode === "current") {
|
||||||
setRgb(
|
setRgb(
|
||||||
colors,
|
colors,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,239 @@
|
||||||
|
import { useEffect, useMemo, useState } from "react";
|
||||||
|
import { StatusBadge } from "@nodedc/ui-react";
|
||||||
|
|
||||||
|
import {
|
||||||
|
fetchDatasetNativeScanPreview,
|
||||||
|
type DatasetNativeScanPreview,
|
||||||
|
} from "../core/lidar/datasetGateway";
|
||||||
|
import {
|
||||||
|
LidarGroundPointCloud,
|
||||||
|
type LidarGroundViewMode,
|
||||||
|
} from "./LidarGroundPointCloud";
|
||||||
|
|
||||||
|
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
|
||||||
|
["intensity", "Исходный скан"],
|
||||||
|
["semantic", "Классы RELLIS"],
|
||||||
|
["ground-truth", "Ground target"],
|
||||||
|
];
|
||||||
|
|
||||||
|
function formatPercent(value: number): string {
|
||||||
|
return new Intl.NumberFormat("ru-RU", {
|
||||||
|
style: "percent",
|
||||||
|
maximumFractionDigits: 1,
|
||||||
|
}).format(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error && error.message.trim()
|
||||||
|
? error.message
|
||||||
|
: "Официальный пример RELLIS недоступен.";
|
||||||
|
}
|
||||||
|
|
||||||
|
function modeExplanation(mode: LidarGroundViewMode): string {
|
||||||
|
if (mode === "semantic") {
|
||||||
|
return "Цвета из закреплённой официальной ontology RELLIS-3D.";
|
||||||
|
}
|
||||||
|
if (mode === "ground-truth") {
|
||||||
|
return "Зелёный — ground, серый — non-ground, янтарный — ignore.";
|
||||||
|
}
|
||||||
|
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function RellisDatasetReview() {
|
||||||
|
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
setError(null);
|
||||||
|
void fetchDatasetNativeScanPreview({
|
||||||
|
sourceId: "rellis-3d/v1.1",
|
||||||
|
signal: controller.signal,
|
||||||
|
})
|
||||||
|
.then((value) => {
|
||||||
|
if (!controller.signal.aborted) setPreview(value);
|
||||||
|
})
|
||||||
|
.catch((loadError: unknown) => {
|
||||||
|
if (!controller.signal.aborted) setError(errorMessage(loadError));
|
||||||
|
});
|
||||||
|
return () => controller.abort();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const scene = useMemo(() => {
|
||||||
|
if (!preview) return null;
|
||||||
|
const empty = Array.from({ length: preview.pointCount }, () => 0);
|
||||||
|
return {
|
||||||
|
pointCount: preview.pointCount,
|
||||||
|
pointsXyzM: preview.pointsXyzM,
|
||||||
|
intensity0To255: preview.remission0To255,
|
||||||
|
semanticRgb0To255: preview.semanticRgb0To255,
|
||||||
|
masks: {
|
||||||
|
currentGround: empty,
|
||||||
|
currentAssigned: preview.evaluationMask,
|
||||||
|
candidateGround: empty,
|
||||||
|
candidateAssigned: preview.evaluationMask,
|
||||||
|
disagreement: empty,
|
||||||
|
groundTruthGround: preview.groundTruthGround,
|
||||||
|
evaluationMask: preview.evaluationMask,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}, [preview]);
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<section className="polygon-review-unavailable">
|
||||||
|
<StatusBadge tone="danger">Smoke preview недоступен</StatusBadge>
|
||||||
|
<h3>RELLIS-3D не открыт</h3>
|
||||||
|
<p>{error}</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!preview || !scene) {
|
||||||
|
return (
|
||||||
|
<section className="polygon-review-unavailable">
|
||||||
|
<StatusBadge tone="accent">Проверяем пример</StatusBadge>
|
||||||
|
<h3>Открываем официальный RELLIS scan</h3>
|
||||||
|
<p>Исходные dataset bytes не переносятся в браузер.</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const evaluatedPoints = preview.evaluationMask.reduce(
|
||||||
|
(total, value) => total + value,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
const groundPoints = preview.groundTruthGround.reduce(
|
||||||
|
(total, value, index) => total + value * (preview.evaluationMask[index] ?? 0),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
const groundClasses = preview.classes.filter(
|
||||||
|
(item) => item.groundTarget === "ground",
|
||||||
|
);
|
||||||
|
const nonGroundClasses = preview.classes.filter(
|
||||||
|
(item) => item.groundTarget === "non-ground",
|
||||||
|
);
|
||||||
|
const ignoredClasses = preview.classes.filter(
|
||||||
|
(item) => item.groundTarget === "ignore",
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section
|
||||||
|
className="dataset-sequence-review rellis-review"
|
||||||
|
aria-label="Просмотр официального примера RELLIS-3D"
|
||||||
|
>
|
||||||
|
<header className="dataset-sequence-review__truth">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">RELLIS S0 · COMPATIBILITY SMOKE</span>
|
||||||
|
<h2>Официальный Ouster scan читается без конвертации формата</h2>
|
||||||
|
<p>
|
||||||
|
Frame {preview.frameId}: исходные XYZI и point-wise labels остались
|
||||||
|
выровнены. Это проверка reader, осей, цветов и ground-policy — ещё не
|
||||||
|
сравнение алгоритмов и не основание для навигации.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<dl>
|
||||||
|
<div>
|
||||||
|
<dt>Исходных точек</dt>
|
||||||
|
<dd>{preview.sourcePointCount.toLocaleString("ru-RU")}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Browser preview</dt>
|
||||||
|
<dd>{preview.pointCount.toLocaleString("ru-RU")}</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Sensor frame</dt>
|
||||||
|
<dd>{preview.coordinateFrame?.frameId ?? "—"}</dd>
|
||||||
|
</div>
|
||||||
|
</dl>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section className="polygon-review-player">
|
||||||
|
<header className="polygon-review-player__header">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">OFFICIAL EXAMPLE · FRAME 000104</span>
|
||||||
|
<strong>Ouster OS1 64 · X forward · Y left · Z up</strong>
|
||||||
|
</div>
|
||||||
|
<div className="polygon-review-mode" aria-label="Режим RELLIS preview">
|
||||||
|
{modes.map(([viewMode, label]) => (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
key={viewMode}
|
||||||
|
data-active={mode === viewMode ? "true" : undefined}
|
||||||
|
onClick={() => setMode(viewMode)}
|
||||||
|
>
|
||||||
|
{label}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div className="polygon-review-stage">
|
||||||
|
<LidarGroundPointCloud
|
||||||
|
frame={scene}
|
||||||
|
mode={mode}
|
||||||
|
coordinateFrame="sensor-fixed"
|
||||||
|
/>
|
||||||
|
<div className="polygon-review-stage__legend">
|
||||||
|
<strong>{modes.find(([viewMode]) => viewMode === mode)?.[1]}</strong>
|
||||||
|
<span>{modeExplanation(mode)}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div className="rellis-review__metrics">
|
||||||
|
<article>
|
||||||
|
<span>Оценимых точек preview</span>
|
||||||
|
<strong>{formatPercent(evaluatedPoints / preview.pointCount)}</strong>
|
||||||
|
<small>ignore не участвует в метриках</small>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Ground среди оценимых</span>
|
||||||
|
<strong>
|
||||||
|
{formatPercent(evaluatedPoints ? groundPoints / evaluatedPoints : 0)}
|
||||||
|
</strong>
|
||||||
|
<small>версионная mapping policy v1</small>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Классов в этом scan</span>
|
||||||
|
<strong>{preview.classes.length}</strong>
|
||||||
|
<small>из 20 классов ontology</small>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="rellis-review__policy" aria-label="Ground target mapping">
|
||||||
|
<header>
|
||||||
|
<span className="section-eyebrow">GROUND TARGET POLICY V1</span>
|
||||||
|
<h3>Что именно будет оцениваться</h3>
|
||||||
|
<p>
|
||||||
|
Mapping принадлежит Mission Core и версионируется отдельно от
|
||||||
|
ontology датасета. Вода, лужи, void, sky и неопределённый object не
|
||||||
|
засчитываются ни как ground, ни как obstacle.
|
||||||
|
</p>
|
||||||
|
</header>
|
||||||
|
<div>
|
||||||
|
<article>
|
||||||
|
<span>Ground</span>
|
||||||
|
<strong>{groundClasses.map((item) => item.className).join(" · ") || "—"}</strong>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Non-ground</span>
|
||||||
|
<strong>
|
||||||
|
{nonGroundClasses.map((item) => item.className).join(" · ") || "—"}
|
||||||
|
</strong>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Ignore</span>
|
||||||
|
<strong>{ignoredClasses.map((item) => item.className).join(" · ") || "—"}</strong>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<p className="dataset-sequence-review__raw-note">
|
||||||
|
Лицензия RELLIS-3D — CC-BY-NC-SA-3.0: источник годится для независимой
|
||||||
|
исследовательской проверки, но коммерческое использование данных или
|
||||||
|
производных моделей требует отдельного лицензионного решения.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -35,7 +35,7 @@ after(async () => {
|
||||||
|
|
||||||
function catalog(overrides = {}) {
|
function catalog(overrides = {}) {
|
||||||
return {
|
return {
|
||||||
schema_version: "missioncore.dataset-gateway-catalog/v2",
|
schema_version: "missioncore.dataset-gateway-catalog/v3",
|
||||||
access: "read-only",
|
access: "read-only",
|
||||||
storage: {
|
storage: {
|
||||||
configured: false,
|
configured: false,
|
||||||
|
|
@ -49,19 +49,60 @@ function catalog(overrides = {}) {
|
||||||
},
|
},
|
||||||
sources: [{
|
sources: [{
|
||||||
source_id: "goose-3d/v2025-08-22",
|
source_id: "goose-3d/v2025-08-22",
|
||||||
|
source_kind: "goose",
|
||||||
display_name: "GOOSE 3D",
|
display_name: "GOOSE 3D",
|
||||||
role: "primary-offroad-semantic-baseline",
|
role: "primary-offroad-semantic-baseline",
|
||||||
license: "CC-BY-SA-4.0",
|
license: "CC-BY-SA-4.0",
|
||||||
|
commercial_use: "allowed-with-share-alike",
|
||||||
format: "semantickitti-xyzi-label",
|
format: "semantickitti-xyzi-label",
|
||||||
frame_semantics: "one-lidar-revolution",
|
frame_semantics: "one-lidar-revolution",
|
||||||
|
sensor: "VLS-128",
|
||||||
platforms: ["MuCAR-3", "ALICE", "Spot"],
|
platforms: ["MuCAR-3", "ALICE", "Spot"],
|
||||||
annotations: ["semantic-point", "instance-point"],
|
annotations: ["semantic-point", "instance-point"],
|
||||||
superclasses: ["natural-ground", "obstacle"],
|
superclasses: ["natural-ground", "obstacle"],
|
||||||
download: {
|
download: {
|
||||||
automatic: false,
|
automatic: false,
|
||||||
reason: "operator-admitted-large-artifact-only",
|
reason: "operator-admitted-large-artifact-only",
|
||||||
|
smoke_example_mb: null,
|
||||||
|
primary_scan_archive_gb: 3.3,
|
||||||
|
label_archive_gb: null,
|
||||||
|
poses_archive_gb: null,
|
||||||
validation_archive_gb: 3.3,
|
validation_archive_gb: 3.3,
|
||||||
},
|
},
|
||||||
|
statistics: {
|
||||||
|
fragment_count: 8,
|
||||||
|
annotated_scan_count: 961,
|
||||||
|
},
|
||||||
|
admission: {
|
||||||
|
status: "blocked-storage-policy",
|
||||||
|
archive: null,
|
||||||
|
frame: null,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
source_id: "rellis-3d/v1.1",
|
||||||
|
source_kind: "rellis",
|
||||||
|
display_name: "RELLIS-3D",
|
||||||
|
role: "independent-offroad-cross-dataset-check",
|
||||||
|
license: "CC-BY-NC-SA-3.0",
|
||||||
|
commercial_use: "research-only-license-review-required",
|
||||||
|
format: "semantickitti-xyzi-label",
|
||||||
|
frame_semantics: "one-lidar-revolution",
|
||||||
|
sensor: "Ouster OS1 64",
|
||||||
|
platforms: ["Clearpath Warthog"],
|
||||||
|
annotations: ["semantic-point"],
|
||||||
|
superclasses: ["ground", "obstacle", "ignore"],
|
||||||
|
download: {
|
||||||
|
automatic: false,
|
||||||
|
reason: "operator-admitted-large-artifact-only",
|
||||||
|
smoke_example_mb: 24,
|
||||||
|
primary_scan_archive_gb: 14,
|
||||||
|
label_archive_gb: 0.174,
|
||||||
|
poses_archive_gb: 0.174,
|
||||||
|
},
|
||||||
|
statistics: {
|
||||||
|
fragment_count: 5,
|
||||||
|
annotated_scan_count: 13_556,
|
||||||
|
},
|
||||||
admission: {
|
admission: {
|
||||||
status: "blocked-storage-policy",
|
status: "blocked-storage-policy",
|
||||||
archive: null,
|
archive: null,
|
||||||
|
|
@ -115,7 +156,8 @@ test("parses three distinct LiDAR representations and current-input boundary", (
|
||||||
);
|
);
|
||||||
assert.equal(parsed.representations[2].accumulation, true);
|
assert.equal(parsed.representations[2].accumulation, true);
|
||||||
assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
|
assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
|
||||||
assert.equal(parsed.source.frameSemantics, "one-lidar-revolution");
|
assert.equal(parsed.sources[0].frameSemantics, "one-lidar-revolution");
|
||||||
|
assert.equal(parsed.sources[1].sourceKind, "rellis");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("rejects path exposure and upgraded K1 semantics", () => {
|
test("rejects path exposure and upgraded K1 semantics", () => {
|
||||||
|
|
@ -147,7 +189,7 @@ test("fetches the read-only gateway endpoint", async () => {
|
||||||
});
|
});
|
||||||
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
|
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
|
||||||
assert.equal(calls[0].init.method, "GET");
|
assert.equal(calls[0].init.method, "GET");
|
||||||
assert.equal(parsed.source.displayName, "GOOSE 3D");
|
assert.equal(parsed.sources[0].displayName, "GOOSE 3D");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("decodes one bounded point-aligned native-scan preview", async () => {
|
test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||||
|
|
@ -182,10 +224,19 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||||
assert.equal(parsed.pointCount, 2);
|
assert.equal(parsed.pointCount, 2);
|
||||||
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
|
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
|
||||||
|
|
||||||
|
let requestedUrl = null;
|
||||||
const fetched = await fetchDatasetNativeScanPreview({
|
const fetched = await fetchDatasetNativeScanPreview({
|
||||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
fetcher: async (url) => {
|
||||||
|
requestedUrl = url;
|
||||||
|
return new Response(JSON.stringify(payload), { status: 200 });
|
||||||
|
},
|
||||||
});
|
});
|
||||||
assert.equal(fetched.frameId, "frame-1");
|
assert.equal(fetched.frameId, "frame-1");
|
||||||
|
assert.deepEqual(fetched.evaluationMask, [1, 1]);
|
||||||
|
assert.equal(
|
||||||
|
requestedUrl,
|
||||||
|
"/api/v1/lidar/dataset-gateway/preview?source_id=goose-3d%2Fv2025-08-22",
|
||||||
|
);
|
||||||
|
|
||||||
payload.semantic_rgb_0_to_255.pop();
|
payload.semantic_rgb_0_to_255.pop();
|
||||||
assert.throws(
|
assert.throws(
|
||||||
|
|
@ -194,6 +245,92 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("decodes the RELLIS compatibility smoke with explicit ignore mask", async () => {
|
||||||
|
const payload = {
|
||||||
|
schema_version: "missioncore.dataset-native-scan-preview/v2",
|
||||||
|
source_id: "rellis-3d/v1.1",
|
||||||
|
frame_id: "000104",
|
||||||
|
representation: "native-scan",
|
||||||
|
sampling: "deterministic-even-index",
|
||||||
|
source_point_count: 3,
|
||||||
|
point_count: 3,
|
||||||
|
points_xyz_m: [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
|
||||||
|
remission_0_to_255: [0, 127, 255],
|
||||||
|
semantic_label_ids: [3, 4, 31],
|
||||||
|
semantic_rgb_0_to_255: [0, 102, 0, 0, 255, 0, 239, 255, 134],
|
||||||
|
ground_truth_ground: [1, 0, 0],
|
||||||
|
evaluation_mask: [1, 1, 0],
|
||||||
|
classes: [
|
||||||
|
{
|
||||||
|
label_id: 3,
|
||||||
|
class_name: "grass",
|
||||||
|
hex: "#006600",
|
||||||
|
ground_target: "ground",
|
||||||
|
source_point_count: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label_id: 4,
|
||||||
|
class_name: "tree",
|
||||||
|
hex: "#00ff00",
|
||||||
|
ground_target: "non-ground",
|
||||||
|
source_point_count: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label_id: 31,
|
||||||
|
class_name: "puddle",
|
||||||
|
hex: "#efff86",
|
||||||
|
ground_target: "ignore",
|
||||||
|
source_point_count: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
coordinate_frame: {
|
||||||
|
frame_id: "sensor/lidar/os1",
|
||||||
|
handedness: "right",
|
||||||
|
x: "forward",
|
||||||
|
y: "left",
|
||||||
|
z: "up",
|
||||||
|
transform_applied: false,
|
||||||
|
},
|
||||||
|
ground_policy: {
|
||||||
|
schema_version: "missioncore.rellis-ground-target-policy/v1",
|
||||||
|
ground: ["grass"],
|
||||||
|
non_ground: ["tree"],
|
||||||
|
ignore: ["puddle"],
|
||||||
|
},
|
||||||
|
source_evidence: {
|
||||||
|
repository_url: "https://github.com/unmannedlab/RELLIS-3D",
|
||||||
|
repository_commit: "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
|
||||||
|
label_config_sha256: "5".repeat(64),
|
||||||
|
point_sha256: "e".repeat(64),
|
||||||
|
label_sha256: "9".repeat(64),
|
||||||
|
license: "CC-BY-NC-SA-3.0",
|
||||||
|
},
|
||||||
|
safety: {
|
||||||
|
visualization_only: true,
|
||||||
|
compatibility_smoke_only: true,
|
||||||
|
navigation_or_safety_accepted: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const parsed = parseDatasetNativeScanPreview(payload);
|
||||||
|
assert.equal(parsed.sourceId, "rellis-3d/v1.1");
|
||||||
|
assert.deepEqual(parsed.evaluationMask, [1, 1, 0]);
|
||||||
|
assert.equal(parsed.classes[2].groundTarget, "ignore");
|
||||||
|
assert.equal(parsed.coordinateFrame.frameId, "sensor/lidar/os1");
|
||||||
|
|
||||||
|
let requestedUrl = null;
|
||||||
|
await fetchDatasetNativeScanPreview({
|
||||||
|
sourceId: "rellis-3d/v1.1",
|
||||||
|
fetcher: async (url) => {
|
||||||
|
requestedUrl = url;
|
||||||
|
return new Response(JSON.stringify(payload), { status: 200 });
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(
|
||||||
|
requestedUrl,
|
||||||
|
"/api/v1/lidar/dataset-gateway/preview?source_id=rellis-3d%2Fv1.1",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
|
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
|
||||||
const payload = {
|
const payload = {
|
||||||
schema_version: "missioncore.dataset-ground-comparison-preview/v2",
|
schema_version: "missioncore.dataset-ground-comparison-preview/v2",
|
||||||
|
|
|
||||||
|
|
@ -203,6 +203,14 @@ qualification bootstrap, never the product data plane. A shared
|
||||||
operator-facing deployment still requires authentication/RBAC, service
|
operator-facing deployment still requires authentication/RBAC, service
|
||||||
supervision and an accepted routed worker transport.
|
supervision and an accepted routed worker transport.
|
||||||
|
|
||||||
|
The Dataset Gateway is now multi-source. RELLIS S0 pins official repository
|
||||||
|
frame `000104`, verifies `131,072` Ouster XYZI points against aligned labels and
|
||||||
|
publishes a bounded semantic/ground-target preview beside GOOSE in the same
|
||||||
|
`Полигон → Датасеты` workflow. This closes only reader, axes, ontology,
|
||||||
|
licensing and versioned ignore-policy compatibility. Full RELLIS Ouster scans,
|
||||||
|
labels and poses remain D-only admission work before the cross-dataset
|
||||||
|
Current/Patchwork++ decision.
|
||||||
|
|
||||||
Mission Core remains a distributed web product. React is served by a long-lived
|
Mission Core remains a distributed web product. React is served by a long-lived
|
||||||
Gateway/API and renders canonical state through browser-native components.
|
Gateway/API and renders canonical state through browser-native components.
|
||||||
Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the
|
Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the
|
||||||
|
|
|
||||||
|
|
@ -878,6 +878,11 @@ workflow:
|
||||||
- `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is
|
- `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is
|
||||||
explicitly grouped into eight independent fragments; Play is bounded to one
|
explicitly grouped into eight independent fragments; Play is bounded to one
|
||||||
fragment and described as accelerated review of sparse annotated scans.
|
fragment and described as accelerated review of sparse annotated scans.
|
||||||
|
- The catalog is multi-source. RELLIS-3D v1.1 appears as an independent Ouster
|
||||||
|
domain with its own license and admission state. Its current `smoke-ready`
|
||||||
|
view shows official frame `000104`, semantic colors, FLU axes and the
|
||||||
|
versioned `ground / non-ground / ignore` mapping; it does not show invented
|
||||||
|
algorithm metrics.
|
||||||
- The dataset viewer exposes source frame number, nanosecond timestamp gaps,
|
- The dataset viewer exposes source frame number, nanosecond timestamp gaps,
|
||||||
ground truth and Current/Patchwork++ overlays.
|
ground truth and Current/Patchwork++ overlays.
|
||||||
- The fixed sensor-centric coordinate frame and operator camera are preserved
|
- The fixed sensor-centric coordinate frame and operator camera are preserved
|
||||||
|
|
@ -894,6 +899,12 @@ The accepted `3.3 GB` annotated validation ZIP is not a continuous recording.
|
||||||
Continuous ego-motion requires a separately admitted raw ROS bag and
|
Continuous ego-motion requires a separately admitted raw ROS bag and
|
||||||
localization source.
|
localization source.
|
||||||
|
|
||||||
|
RELLIS S0 uses only a bounded derivative of the pinned official example for
|
||||||
|
browser inspection. The full `14 GB` Ouster SemanticKITTI scans, labels and
|
||||||
|
poses remain a separate D-only admission gate. Its `CC-BY-NC-SA-3.0` license
|
||||||
|
means cross-dataset research evidence cannot silently become an unrestricted
|
||||||
|
commercial runtime or training dependency.
|
||||||
|
|
||||||
The backend reads the configured repository from
|
The backend reads the configured repository from
|
||||||
`MISSIONCORE_POLYGON_RUNS_ROOT` using
|
`MISSIONCORE_POLYGON_RUNS_ROOT` using
|
||||||
`QualificationRunStore(read_only=True)`. It exposes only:
|
`QualificationRunStore(read_only=True)`. It exposes only:
|
||||||
|
|
|
||||||
|
|
@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
|
||||||
domain adaptation after a public baseline proves that the pipeline and metric
|
domain adaptation after a public baseline proves that the pipeline and metric
|
||||||
harness work.
|
harness work.
|
||||||
|
|
||||||
### L2.5 — Dataset Gateway — first-frame A/B complete
|
### L2.5 — Dataset Gateway — GOOSE qualification and RELLIS S0 complete
|
||||||
|
|
||||||
- [x] Define separate `native-scan`, `normalized-scan` and
|
- [x] Define separate `native-scan`, `normalized-scan` and
|
||||||
`rolling-local-map` representations.
|
`rolling-local-map` representations.
|
||||||
|
|
@ -317,12 +317,21 @@ harness work.
|
||||||
through an immutable R1 Polygon replay-shadow run.
|
through an immutable R1 Polygon replay-shadow run.
|
||||||
- [x] Add named range/FOV/density/noise/dropout degradation profiles without
|
- [x] Add named range/FOV/density/noise/dropout degradation profiles without
|
||||||
overwriting the native frame.
|
overwriting the native frame.
|
||||||
|
- [x] Upgrade the gateway to two independently identified sources instead of
|
||||||
|
treating GOOSE as a universal dataset contract.
|
||||||
|
- [x] Pin and verify the official RELLIS-3D Ouster example, preserve its
|
||||||
|
`131,072` point/label alignment and publish a bounded semantic viewer.
|
||||||
|
- [x] Version the RELLIS `ground / non-ground / ignore` mapping and expose
|
||||||
|
licensing as research-only evidence.
|
||||||
|
- [ ] Admit the full RELLIS Ouster SemanticKITTI scans, labels and poses on
|
||||||
|
worker D, then run the same Current/Patchwork++ harness.
|
||||||
|
|
||||||
The architectural contract and run sequence are fixed in
|
The architectural contract and run sequence are fixed in
|
||||||
`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its
|
`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its
|
||||||
published 3D format is one LiDAR revolution with point-wise semantic and
|
published 3D format is one LiDAR revolution with point-wise semantic and
|
||||||
instance labels in off-road environments. RELLIS-3D remains the second-source
|
instance labels in off-road environments. The RELLIS-3D compatibility smoke is
|
||||||
cross-check after the GOOSE harness is stable.
|
now complete against official frame `000104`; the full second-source algorithm
|
||||||
|
cross-check remains the next gate.
|
||||||
|
|
||||||
### L3 — LiDAR-native 3D detection
|
### L3 — LiDAR-native 3D detection
|
||||||
|
|
||||||
|
|
@ -394,7 +403,10 @@ The near-term value is not a prettier point cloud:
|
||||||
The GOOSE validation-split and deterministic degradation gates are complete:
|
The GOOSE validation-split and deterministic degradation gates are complete:
|
||||||
pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises
|
pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises
|
||||||
natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95
|
natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95
|
||||||
across 961 frames. The decision remains shadow-only. The highest-value
|
across 961 frames. RELLIS S0 proves that the shared reader, axes, ontology and
|
||||||
immediate work is now a second-source cross-check plus LiDAR-native detection
|
ground-target mapping work on a different Ouster off-road domain; it does not
|
||||||
on the same gateway. Nvblox and alternative SLAM remain later because their
|
yet score an algorithm. The decision remains shadow-only. The highest-value
|
||||||
timing, pose and scan-geometry gates are not yet satisfied.
|
immediate work is full RELLIS ground qualification, followed by the real
|
||||||
|
sensor input contract and LiDAR-native detection on the same gateway. Nvblox
|
||||||
|
and alternative SLAM remain later because their timing, pose and scan-geometry
|
||||||
|
gates are not yet satisfied.
|
||||||
|
|
|
||||||
|
|
@ -19,12 +19,13 @@ datasets and future real onboard sensors.
|
||||||
|
|
||||||
Implemented now:
|
Implemented now:
|
||||||
|
|
||||||
- `missioncore.dataset-gateway-catalog/v2`, exposed read-only at
|
- multi-source `missioncore.dataset-gateway-catalog/v3`, exposed read-only at
|
||||||
`GET /api/v1/lidar/dataset-gateway`;
|
`GET /api/v1/lidar/dataset-gateway`;
|
||||||
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a
|
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a
|
||||||
static React status;
|
static React status;
|
||||||
- a bounded `missioncore.dataset-native-scan-preview/v1`, exposed read-only at
|
- bounded `missioncore.dataset-native-scan-preview/v1` (GOOSE) and `/v2`
|
||||||
`GET /api/v1/lidar/dataset-gateway/preview`;
|
(RELLIS smoke) artifacts, selected by source at
|
||||||
|
`GET /api/v1/lidar/dataset-gateway/preview?source_id=...`;
|
||||||
- explicit `native-scan`, `normalized-scan` and `rolling-local-map`
|
- explicit `native-scan`, `normalized-scan` and `rolling-local-map`
|
||||||
representations;
|
representations;
|
||||||
- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI
|
- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI
|
||||||
|
|
@ -39,6 +40,10 @@ Implemented now:
|
||||||
`Открыть` action;
|
`Открыть` action;
|
||||||
- one admitted GOOSE validation frame with source colors, normalized remission
|
- one admitted GOOSE validation frame with source colors, normalized remission
|
||||||
and an independent ground-truth view;
|
and an independent ground-truth view;
|
||||||
|
- a pinned RELLIS-3D v1.1 compatibility smoke using the official Ouster example
|
||||||
|
frame `000104`, its native `131,072` XYZI points, point-aligned labels,
|
||||||
|
official ontology colors, explicit FLU sensor axes and a versioned
|
||||||
|
`ground / non-ground / ignore` evaluation policy;
|
||||||
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
||||||
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
||||||
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
||||||
|
|
@ -54,6 +59,8 @@ Not implemented:
|
||||||
- no implicit coordinate conversion;
|
- no implicit coordinate conversion;
|
||||||
- no fake ring/timestamp reconstruction for K1 MQTT evidence;
|
- no fake ring/timestamp reconstruction for K1 MQTT evidence;
|
||||||
- no model training or production promotion;
|
- no model training or production promotion;
|
||||||
|
- no full RELLIS Ouster archive, pose set, algorithm comparison or ROS bag
|
||||||
|
admission yet;
|
||||||
- no rolling-map implementation yet.
|
- no rolling-map implementation yet.
|
||||||
|
|
||||||
## Product surface boundary
|
## Product surface boundary
|
||||||
|
|
@ -73,10 +80,11 @@ The gateway is not part of device quality diagnostics.
|
||||||
backend evidence contract. They are not a second operator navigation item
|
backend evidence contract. They are not a second operator navigation item
|
||||||
and are not presented as vehicle motion.
|
and are not presented as vehicle motion.
|
||||||
|
|
||||||
Before real dataset bytes exist, the catalog explains the blocked storage gate
|
Before compatible source evidence exists, the catalog explains the blocked
|
||||||
and keeps `Открыть датасет` disabled. It becomes available
|
storage gate and keeps `Открыть датасет` disabled. GOOSE becomes available only
|
||||||
only after the worker manifest says `frame-ready` and the bounded preview passes
|
after the worker manifest says `frame-ready`; RELLIS becomes available at
|
||||||
its own point-alignment and safety checks.
|
`smoke-ready` only for the pinned official example. Both bounded previews must
|
||||||
|
pass their own identity, point-alignment and safety checks.
|
||||||
|
|
||||||
## Why public recordings look different
|
## Why public recordings look different
|
||||||
|
|
||||||
|
|
@ -191,8 +199,53 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
||||||
- [x] Patchwork++ one-frame accuracy measured against ground truth.
|
- [x] Patchwork++ one-frame accuracy measured against ground truth.
|
||||||
- [x] Current/Patchwork++ validation-split gate qualified.
|
- [x] Current/Patchwork++ validation-split gate qualified.
|
||||||
- [x] Sensor-degradation matrix qualified.
|
- [x] Sensor-degradation matrix qualified.
|
||||||
|
- [x] Dataset catalog accepts more than one independently identified source.
|
||||||
|
- [x] Official RELLIS Ouster example passes the shared SemanticKITTI reader.
|
||||||
|
- [x] RELLIS axes, ontology colors and versioned ground-target/ignore mapping
|
||||||
|
are visible in the same Dataset workflow.
|
||||||
|
- [ ] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
|
||||||
|
worker D.
|
||||||
|
- [ ] Current/Patchwork++ RELLIS comparison qualified without obstacle loss.
|
||||||
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
||||||
|
|
||||||
|
## RELLIS S0 compatibility evidence
|
||||||
|
|
||||||
|
RELLIS is the independent second-source check, not extra decoration in the
|
||||||
|
catalog. GOOSE and RELLIS share the ingress format and viewer, but retain
|
||||||
|
separate source identity, sensor domain, ontology, licensing and results.
|
||||||
|
|
||||||
|
The pinned smoke input is the official repository example at commit
|
||||||
|
`c17a118fcaed1559f03cc32cc3a91dedc557f8b8`:
|
||||||
|
|
||||||
|
- point file: `utils/example/000104.bin`;
|
||||||
|
- label file: `utils/example/000104.label`;
|
||||||
|
- source points: `131,072`;
|
||||||
|
- point SHA-256:
|
||||||
|
`ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af`;
|
||||||
|
- label SHA-256:
|
||||||
|
`9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e`;
|
||||||
|
- official ontology config SHA-256:
|
||||||
|
`573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad`;
|
||||||
|
- bounded browser preview: `20,000` deterministic even-index points;
|
||||||
|
- coordinate frame: right-handed, `x` forward, `y` left, `z` up; no vehicle
|
||||||
|
transform, pose registration or deskew is claimed;
|
||||||
|
- license: `CC-BY-NC-SA-3.0`; this is a research qualification source, not an
|
||||||
|
unrestricted commercial runtime dependency.
|
||||||
|
|
||||||
|
Mission Core ground policy v1 treats `dirt`, `grass`, `asphalt`, `concrete` and
|
||||||
|
`mud` as ground; clear structures, vegetation, people, vehicles and rubble as
|
||||||
|
non-ground; and `void`, `water`, `sky`, generic `object` and `puddle` as
|
||||||
|
`ignore`. The ignore set is excluded from future metrics instead of being
|
||||||
|
silently counted as either free ground or obstacle. This mapping is a Mission
|
||||||
|
Core evaluation decision, not an upstream RELLIS claim.
|
||||||
|
|
||||||
|
The smoke closes reader, alignment, axes, palette and mapping compatibility
|
||||||
|
only. The next gate admits the `14 GB` Ouster SemanticKITTI scans plus the
|
||||||
|
`174 MB` labels and `174 MB` poses to D-only worker storage. Full ROS bags are
|
||||||
|
not needed for the ground cross-check. Patchwork++ may be compared only after a
|
||||||
|
physical Ouster mounting-height profile is evidenced; no height is inferred
|
||||||
|
from the example cloud.
|
||||||
|
|
||||||
## First GOOSE admission evidence
|
## First GOOSE admission evidence
|
||||||
|
|
||||||
- worker root: canonical D-only root, path never exposed by the HTTP API;
|
- worker root: canonical D-only root, path never exposed by the HTTP API;
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,12 @@ first admitted source because it publishes off-road point-wise semantic and
|
||||||
instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its
|
instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its
|
||||||
annotated point-cloud file represents one LiDAR revolution.
|
annotated point-cloud file represents one LiDAR revolution.
|
||||||
|
|
||||||
|
The gateway contract is multi-source, not GOOSE-shaped. RELLIS-3D v1.1 is the
|
||||||
|
second source: it uses the same lossless SemanticKITTI ingress but keeps a
|
||||||
|
separate Ouster OS1 domain, ontology, admission state, ground-target mapping,
|
||||||
|
license and result set. Shared format never implies shared calibration,
|
||||||
|
mounting height or commercial-use rights.
|
||||||
|
|
||||||
The gateway:
|
The gateway:
|
||||||
|
|
||||||
- preserves the native GOOSE frame before adaptation;
|
- preserves the native GOOSE frame before adaptation;
|
||||||
|
|
@ -48,6 +54,10 @@ The gateway:
|
||||||
- refuses automatic downloads of large archives;
|
- refuses automatic downloads of large archives;
|
||||||
- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror;
|
- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror;
|
||||||
- never promotes K1 `lio_pcl` to `native-scan`.
|
- never promotes K1 `lio_pcl` to `native-scan`.
|
||||||
|
- excludes explicitly ambiguous RELLIS classes through a versioned `ignore`
|
||||||
|
mask instead of silently scoring them as ground or obstacle;
|
||||||
|
- exposes the RELLIS `CC-BY-NC-SA-3.0` restriction as research evidence and
|
||||||
|
never treats it as an unrestricted production dependency.
|
||||||
|
|
||||||
Large-artifact execution remains worker-local. The browser receives only a
|
Large-artifact execution remains worker-local. The browser receives only a
|
||||||
path-free admission manifest and a bounded visualization preview. The canonical
|
path-free admission manifest and a bounded visualization preview. The canonical
|
||||||
|
|
@ -82,6 +92,8 @@ unavailable rather than inventing timestamps.
|
||||||
- Until bytes are installed, the UI exposes an explicit empty state. After a
|
- Until bytes are installed, the UI exposes an explicit empty state. After a
|
||||||
`frame-ready` worker admission, the same surface becomes an interactive
|
`frame-ready` worker admission, the same surface becomes an interactive
|
||||||
source/remission/ground-truth viewer without changing the representation.
|
source/remission/ground-truth viewer without changing the representation.
|
||||||
|
- A bounded `smoke-ready` source may prove reader, axes, ontology and mapping
|
||||||
|
compatibility without claiming full-dataset or algorithm qualification.
|
||||||
- Dataset and device inputs can share downstream algorithms only after their
|
- Dataset and device inputs can share downstream algorithms only after their
|
||||||
normalized contracts match.
|
normalized contracts match.
|
||||||
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
||||||
|
|
@ -115,6 +127,7 @@ unavailable rather than inventing timestamps.
|
||||||
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
|
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
|
||||||
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
|
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
|
||||||
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
|
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
|
||||||
|
- [RELLIS-3D official repository](https://github.com/unmannedlab/RELLIS-3D)
|
||||||
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
||||||
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
||||||
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,15 @@ from k1link.datasets.goose_qualification import (
|
||||||
GroundAcceptancePolicy,
|
GroundAcceptancePolicy,
|
||||||
qualify_goose_ground,
|
qualify_goose_ground,
|
||||||
)
|
)
|
||||||
|
from k1link.datasets.rellis_smoke import (
|
||||||
|
RELLIS_CLASSES,
|
||||||
|
RELLIS_GROUND_POLICY_SCHEMA,
|
||||||
|
RELLIS_PREVIEW_SCHEMA,
|
||||||
|
RELLIS_SOURCE_ID,
|
||||||
|
RellisSmokeError,
|
||||||
|
build_rellis_official_smoke_preview,
|
||||||
|
rellis_native_scan_preview,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"DATASET_GATEWAY_CATALOG_SCHEMA",
|
"DATASET_GATEWAY_CATALOG_SCHEMA",
|
||||||
|
|
@ -54,12 +63,18 @@ __all__ = [
|
||||||
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
|
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
|
||||||
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
||||||
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
||||||
|
"RELLIS_CLASSES",
|
||||||
|
"RELLIS_GROUND_POLICY_SCHEMA",
|
||||||
|
"RELLIS_PREVIEW_SCHEMA",
|
||||||
|
"RELLIS_SOURCE_ID",
|
||||||
|
"RellisSmokeError",
|
||||||
"GoosePatchworkProfile",
|
"GoosePatchworkProfile",
|
||||||
"GroundAcceptancePolicy",
|
"GroundAcceptancePolicy",
|
||||||
"DegradationProfile",
|
"DegradationProfile",
|
||||||
"DEFAULT_DEGRADATIONS",
|
"DEFAULT_DEGRADATIONS",
|
||||||
"benchmark_goose_current_ground",
|
"benchmark_goose_current_ground",
|
||||||
"benchmark_goose_patchwork_ground",
|
"benchmark_goose_patchwork_ground",
|
||||||
|
"build_rellis_official_smoke_preview",
|
||||||
"configured_dataset_admission_manifest",
|
"configured_dataset_admission_manifest",
|
||||||
"configured_dataset_ground_preview",
|
"configured_dataset_ground_preview",
|
||||||
"configured_dataset_preview",
|
"configured_dataset_preview",
|
||||||
|
|
@ -69,5 +84,6 @@ __all__ = [
|
||||||
"read_dataset_ground_preview",
|
"read_dataset_ground_preview",
|
||||||
"read_dataset_native_scan_preview",
|
"read_dataset_native_scan_preview",
|
||||||
"read_semantic_kitti_frame",
|
"read_semantic_kitti_frame",
|
||||||
|
"rellis_native_scan_preview",
|
||||||
"qualify_goose_ground",
|
"qualify_goose_ground",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
|
||||||
)
|
)
|
||||||
from k1link.datasets.goose_qualification import qualify_goose_ground
|
from k1link.datasets.goose_qualification import qualify_goose_ground
|
||||||
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
||||||
|
from k1link.datasets.rellis_smoke import (
|
||||||
|
RellisSmokeError,
|
||||||
|
build_rellis_official_smoke_preview,
|
||||||
|
)
|
||||||
|
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
add_completion=False,
|
add_completion=False,
|
||||||
|
|
@ -158,5 +162,50 @@ def build_goose_ground_review_command(
|
||||||
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
@app.command("build-rellis-smoke-preview")
|
||||||
|
def build_rellis_smoke_preview_command(
|
||||||
|
points: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--points", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
labels: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--labels", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
out: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--out", dir_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
preview_points: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--preview-points", min=1, max=50_000),
|
||||||
|
] = 20_000,
|
||||||
|
) -> None:
|
||||||
|
"""Verify the pinned official RELLIS example and publish a bounded preview."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
preview = build_rellis_official_smoke_preview(
|
||||||
|
points,
|
||||||
|
labels,
|
||||||
|
out,
|
||||||
|
preview_points=preview_points,
|
||||||
|
)
|
||||||
|
except RellisSmokeError as exc:
|
||||||
|
typer.echo(str(exc), err=True)
|
||||||
|
raise typer.Exit(code=2) from exc
|
||||||
|
typer.echo(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"source_id": preview["source_id"],
|
||||||
|
"frame_id": preview["frame_id"],
|
||||||
|
"source_point_count": preview["source_point_count"],
|
||||||
|
"preview_point_count": preview["point_count"],
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app()
|
app()
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ import numpy.typing as npt
|
||||||
|
|
||||||
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
|
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
|
||||||
|
|
||||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v3"
|
||||||
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||||
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||||
|
|
@ -177,6 +177,7 @@ def _is_worker_d_storage(root: Path | None) -> bool:
|
||||||
def dataset_gateway_catalog(
|
def dataset_gateway_catalog(
|
||||||
dataset_root: Path | None = None,
|
dataset_root: Path | None = None,
|
||||||
admission_manifest_path: Path | None = None,
|
admission_manifest_path: Path | None = None,
|
||||||
|
rellis_preview_path: Path | None = None,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||||
|
|
||||||
|
|
@ -196,6 +197,14 @@ def dataset_gateway_catalog(
|
||||||
locally_admitted = _is_worker_d_storage(root)
|
locally_admitted = _is_worker_d_storage(root)
|
||||||
worker_admitted = bool(admission and admission["storage"]["admitted"])
|
worker_admitted = bool(admission and admission["storage"]["admitted"])
|
||||||
storage_admitted = locally_admitted or worker_admitted
|
storage_admitted = locally_admitted or worker_admitted
|
||||||
|
rellis_preview: dict[str, Any] | None = None
|
||||||
|
if rellis_preview_path is not None and rellis_preview_path.is_file():
|
||||||
|
try:
|
||||||
|
candidate = read_dataset_native_scan_preview(rellis_preview_path)
|
||||||
|
if candidate.get("source_id") == "rellis-3d/v1.1":
|
||||||
|
rellis_preview = candidate
|
||||||
|
except DatasetAdmissionError:
|
||||||
|
pass
|
||||||
source_status = (
|
source_status = (
|
||||||
str(admission["status"])
|
str(admission["status"])
|
||||||
if admission is not None
|
if admission is not None
|
||||||
|
|
@ -212,6 +221,20 @@ def dataset_gateway_catalog(
|
||||||
if storage_admitted
|
if storage_admitted
|
||||||
else "configure-dataset-root-on-worker-d"
|
else "configure-dataset-root-on-worker-d"
|
||||||
)
|
)
|
||||||
|
rellis_status = (
|
||||||
|
"smoke-ready"
|
||||||
|
if rellis_preview is not None
|
||||||
|
else "ready-for-smoke"
|
||||||
|
if storage_admitted
|
||||||
|
else "blocked-storage-policy"
|
||||||
|
)
|
||||||
|
catalog_next_action = (
|
||||||
|
"admit-rellis-ouster-semantickitti-to-worker-d"
|
||||||
|
if rellis_preview is not None and source_status == "frame-ready"
|
||||||
|
else "build-rellis-official-example-smoke"
|
||||||
|
if source_status == "frame-ready"
|
||||||
|
else next_action
|
||||||
|
)
|
||||||
return {
|
return {
|
||||||
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
|
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||||
"access": "read-only",
|
"access": "read-only",
|
||||||
|
|
@ -234,11 +257,14 @@ def dataset_gateway_catalog(
|
||||||
"sources": [
|
"sources": [
|
||||||
{
|
{
|
||||||
"source_id": "goose-3d/v2025-08-22",
|
"source_id": "goose-3d/v2025-08-22",
|
||||||
|
"source_kind": "goose",
|
||||||
"display_name": "GOOSE 3D",
|
"display_name": "GOOSE 3D",
|
||||||
"role": "primary-offroad-semantic-baseline",
|
"role": "primary-offroad-semantic-baseline",
|
||||||
"license": "CC-BY-SA-4.0",
|
"license": "CC-BY-SA-4.0",
|
||||||
|
"commercial_use": "allowed-with-share-alike",
|
||||||
"format": "semantickitti-xyzi-label",
|
"format": "semantickitti-xyzi-label",
|
||||||
"frame_semantics": "one-lidar-revolution",
|
"frame_semantics": "one-lidar-revolution",
|
||||||
|
"sensor": "VLS-128",
|
||||||
"platforms": ["MuCAR-3", "ALICE", "Spot"],
|
"platforms": ["MuCAR-3", "ALICE", "Spot"],
|
||||||
"annotations": ["semantic-point", "instance-point"],
|
"annotations": ["semantic-point", "instance-point"],
|
||||||
"superclasses": [
|
"superclasses": [
|
||||||
|
|
@ -255,10 +281,18 @@ def dataset_gateway_catalog(
|
||||||
"download": {
|
"download": {
|
||||||
"automatic": False,
|
"automatic": False,
|
||||||
"reason": "operator-admitted-large-artifact-only",
|
"reason": "operator-admitted-large-artifact-only",
|
||||||
|
"smoke_example_mb": None,
|
||||||
|
"primary_scan_archive_gb": 3.3,
|
||||||
|
"label_archive_gb": None,
|
||||||
|
"poses_archive_gb": None,
|
||||||
"training_archive_gb": 27.0,
|
"training_archive_gb": 27.0,
|
||||||
"validation_archive_gb": 3.3,
|
"validation_archive_gb": 3.3,
|
||||||
"test_archive_gb": 3.3,
|
"test_archive_gb": 3.3,
|
||||||
},
|
},
|
||||||
|
"statistics": {
|
||||||
|
"fragment_count": 8,
|
||||||
|
"annotated_scan_count": 961,
|
||||||
|
},
|
||||||
"admission": {
|
"admission": {
|
||||||
"status": source_status,
|
"status": source_status,
|
||||||
"native_scan": "ready-after-download",
|
"native_scan": "ready-after-download",
|
||||||
|
|
@ -267,7 +301,72 @@ def dataset_gateway_catalog(
|
||||||
"archive": admission["archive"] if admission is not None else None,
|
"archive": admission["archive"] if admission is not None else None,
|
||||||
"frame": admission["frame"] if admission is not None else None,
|
"frame": admission["frame"] if admission is not None else None,
|
||||||
},
|
},
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"source_id": "rellis-3d/v1.1",
|
||||||
|
"source_kind": "rellis",
|
||||||
|
"display_name": "RELLIS-3D",
|
||||||
|
"role": "independent-offroad-cross-dataset-check",
|
||||||
|
"license": "CC-BY-NC-SA-3.0",
|
||||||
|
"commercial_use": "research-only-license-review-required",
|
||||||
|
"format": "semantickitti-xyzi-label",
|
||||||
|
"frame_semantics": "one-lidar-revolution",
|
||||||
|
"sensor": "Ouster OS1 64",
|
||||||
|
"platforms": ["Clearpath Warthog"],
|
||||||
|
"annotations": ["semantic-point"],
|
||||||
|
"superclasses": [
|
||||||
|
"ground",
|
||||||
|
"vegetation",
|
||||||
|
"structure",
|
||||||
|
"obstacle",
|
||||||
|
"vehicle",
|
||||||
|
"human",
|
||||||
|
"water",
|
||||||
|
"ignore",
|
||||||
|
],
|
||||||
|
"download": {
|
||||||
|
"automatic": False,
|
||||||
|
"reason": "operator-admitted-large-artifact-only",
|
||||||
|
"smoke_example_mb": 24.0,
|
||||||
|
"primary_scan_archive_gb": 14.0,
|
||||||
|
"label_archive_gb": 0.174,
|
||||||
|
"poses_archive_gb": 0.174,
|
||||||
|
"training_archive_gb": None,
|
||||||
|
"validation_archive_gb": None,
|
||||||
|
"test_archive_gb": None,
|
||||||
|
},
|
||||||
|
"statistics": {
|
||||||
|
"fragment_count": 5,
|
||||||
|
"annotated_scan_count": 13_556,
|
||||||
|
},
|
||||||
|
"admission": {
|
||||||
|
"status": rellis_status,
|
||||||
|
"native_scan": (
|
||||||
|
"official-example-compatible"
|
||||||
|
if rellis_preview is not None
|
||||||
|
else "requires-official-example-smoke"
|
||||||
|
),
|
||||||
|
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
|
||||||
|
"rolling_local_map": "requires-poses-timing-and-map-policy",
|
||||||
|
"archive": None,
|
||||||
|
"frame": (
|
||||||
|
{
|
||||||
|
"frame_id": rellis_preview["frame_id"],
|
||||||
|
"point_count": rellis_preview["source_point_count"],
|
||||||
|
"semantic_class_count": len(rellis_preview["classes"]),
|
||||||
|
"ground_truth_ground_fraction": (
|
||||||
|
sum(rellis_preview["ground_truth_ground"])
|
||||||
|
/ rellis_preview["point_count"]
|
||||||
|
),
|
||||||
|
"preview_point_count": rellis_preview["point_count"],
|
||||||
|
"preview_sha256": None,
|
||||||
|
"preview_available": True,
|
||||||
|
}
|
||||||
|
if rellis_preview is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"representations": [
|
"representations": [
|
||||||
{
|
{
|
||||||
|
|
@ -330,7 +429,7 @@ def dataset_gateway_catalog(
|
||||||
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
|
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"next_action": next_action,
|
"next_action": catalog_next_action,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -417,9 +516,13 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
||||||
|
|
||||||
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
|
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
|
||||||
|
identity = (document.get("schema_version"), document.get("source_id"))
|
||||||
if (
|
if (
|
||||||
document.get("schema_version") != DATASET_PREVIEW_SCHEMA
|
identity
|
||||||
or document.get("source_id") != "goose-3d/v2025-08-22"
|
not in {
|
||||||
|
(DATASET_PREVIEW_SCHEMA, "goose-3d/v2025-08-22"),
|
||||||
|
("missioncore.dataset-native-scan-preview/v2", "rellis-3d/v1.1"),
|
||||||
|
}
|
||||||
or document.get("representation") != "native-scan"
|
or document.get("representation") != "native-scan"
|
||||||
or document.get("sampling") != "deterministic-even-index"
|
or document.get("sampling") != "deterministic-even-index"
|
||||||
):
|
):
|
||||||
|
|
@ -433,6 +536,11 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
semantic_ids = document.get("semantic_label_ids")
|
semantic_ids = document.get("semantic_label_ids")
|
||||||
semantic_rgb = document.get("semantic_rgb_0_to_255")
|
semantic_rgb = document.get("semantic_rgb_0_to_255")
|
||||||
ground = document.get("ground_truth_ground")
|
ground = document.get("ground_truth_ground")
|
||||||
|
evaluated = (
|
||||||
|
document.get("evaluation_mask")
|
||||||
|
if identity[1] == "rellis-3d/v1.1"
|
||||||
|
else [1] * point_count
|
||||||
|
)
|
||||||
if (
|
if (
|
||||||
not isinstance(points, list)
|
not isinstance(points, list)
|
||||||
or len(points) != point_count
|
or len(points) != point_count
|
||||||
|
|
@ -444,6 +552,8 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
or len(semantic_rgb) != point_count * 3
|
or len(semantic_rgb) != point_count * 3
|
||||||
or not isinstance(ground, list)
|
or not isinstance(ground, list)
|
||||||
or len(ground) != point_count
|
or len(ground) != point_count
|
||||||
|
or not isinstance(evaluated, list)
|
||||||
|
or len(evaluated) != point_count
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
|
raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
|
||||||
for point in points:
|
for point in points:
|
||||||
|
|
@ -463,6 +573,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
(semantic_ids, 65_535, "semantic label"),
|
(semantic_ids, 65_535, "semantic label"),
|
||||||
(semantic_rgb, 255, "semantic color"),
|
(semantic_rgb, 255, "semantic color"),
|
||||||
(ground, 1, "ground mask"),
|
(ground, 1, "ground mask"),
|
||||||
|
(evaluated, 1, "evaluation mask"),
|
||||||
):
|
):
|
||||||
if any(
|
if any(
|
||||||
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
|
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
|
||||||
|
|
@ -473,16 +584,81 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
if not isinstance(classes, list) or len(classes) > 64:
|
if not isinstance(classes, list) or len(classes) > 64:
|
||||||
raise DatasetAdmissionError("dataset preview class catalog is incompatible")
|
raise DatasetAdmissionError("dataset preview class catalog is incompatible")
|
||||||
safety = _object(document.get("safety"), "safety")
|
safety = _object(document.get("safety"), "safety")
|
||||||
if safety != {
|
expected_safety = {
|
||||||
"visualization_only": True,
|
"visualization_only": True,
|
||||||
"navigation_or_safety_accepted": False,
|
"navigation_or_safety_accepted": False,
|
||||||
}:
|
}
|
||||||
|
if identity[1] == "rellis-3d/v1.1":
|
||||||
|
expected_safety["compatibility_smoke_only"] = True
|
||||||
|
if safety != expected_safety:
|
||||||
raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
|
raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
|
||||||
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
|
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
|
||||||
raise DatasetAdmissionError("dataset preview frame id is incompatible")
|
raise DatasetAdmissionError("dataset preview frame id is incompatible")
|
||||||
|
if identity[1] == "rellis-3d/v1.1":
|
||||||
|
_validate_rellis_preview_metadata(document, classes)
|
||||||
return document
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_rellis_preview_metadata(
|
||||||
|
document: dict[str, Any],
|
||||||
|
classes: list[Any],
|
||||||
|
) -> None:
|
||||||
|
coordinate_frame = _object(document.get("coordinate_frame"), "coordinate_frame")
|
||||||
|
if coordinate_frame != {
|
||||||
|
"frame_id": "sensor/lidar/os1",
|
||||||
|
"handedness": "right",
|
||||||
|
"x": "forward",
|
||||||
|
"y": "left",
|
||||||
|
"z": "up",
|
||||||
|
"transform_applied": False,
|
||||||
|
}:
|
||||||
|
raise DatasetAdmissionError("RELLIS coordinate frame is incompatible")
|
||||||
|
policy = _object(document.get("ground_policy"), "ground_policy")
|
||||||
|
if (
|
||||||
|
policy.get("schema_version") != "missioncore.rellis-ground-target-policy/v1"
|
||||||
|
or set(policy) != {"schema_version", "ground", "non_ground", "ignore"}
|
||||||
|
or any(
|
||||||
|
not isinstance(policy.get(key), list)
|
||||||
|
or any(not isinstance(value, str) or not value for value in policy[key])
|
||||||
|
for key in ("ground", "non_ground", "ignore")
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("RELLIS ground target policy is incompatible")
|
||||||
|
for item in classes:
|
||||||
|
value = _object(item, "class")
|
||||||
|
if (
|
||||||
|
value.get("ground_target") not in {"ground", "non-ground", "ignore"}
|
||||||
|
or not isinstance(value.get("label_id"), int)
|
||||||
|
or not isinstance(value.get("class_name"), str)
|
||||||
|
or not isinstance(value.get("hex"), str)
|
||||||
|
or not isinstance(value.get("source_point_count"), int)
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("RELLIS class catalog is incompatible")
|
||||||
|
evidence = _object(document.get("source_evidence"), "source_evidence")
|
||||||
|
if (
|
||||||
|
set(evidence)
|
||||||
|
!= {
|
||||||
|
"repository_url",
|
||||||
|
"repository_commit",
|
||||||
|
"label_config_sha256",
|
||||||
|
"point_sha256",
|
||||||
|
"label_sha256",
|
||||||
|
"license",
|
||||||
|
}
|
||||||
|
or evidence.get("repository_url") != "https://github.com/unmannedlab/RELLIS-3D"
|
||||||
|
or evidence.get("repository_commit")
|
||||||
|
!= "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
|
||||||
|
or evidence.get("label_config_sha256")
|
||||||
|
!= "573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||||
|
or evidence.get("point_sha256")
|
||||||
|
!= "ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||||
|
or evidence.get("label_sha256")
|
||||||
|
!= "9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||||
|
or evidence.get("license") != "CC-BY-NC-SA-3.0"
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("RELLIS source evidence is incompatible")
|
||||||
|
|
||||||
|
|
||||||
def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
||||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
|
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
|
||||||
if (
|
if (
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,275 @@
|
||||||
|
"""Pinned RELLIS-3D compatibility smoke for the shared Dataset Gateway.
|
||||||
|
|
||||||
|
The smoke check deliberately stops before algorithm qualification. It proves
|
||||||
|
that Mission Core can read one official Ouster OS1 SemanticKITTI frame, preserve
|
||||||
|
point/label alignment, apply an explicit ground-evaluation policy and publish a
|
||||||
|
bounded visualization artifact. Full RELLIS archives and ROS bags remain
|
||||||
|
worker-D-only inputs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
from collections import Counter
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Final, Literal
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from k1link.datasets.gateway import DatasetFrameError, DatasetPointFrame, read_semantic_kitti_frame
|
||||||
|
|
||||||
|
RELLIS_SOURCE_ID: Final = "rellis-3d/v1.1"
|
||||||
|
RELLIS_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v2"
|
||||||
|
RELLIS_GROUND_POLICY_SCHEMA: Final = "missioncore.rellis-ground-target-policy/v1"
|
||||||
|
RELLIS_REPOSITORY_URL: Final = "https://github.com/unmannedlab/RELLIS-3D"
|
||||||
|
RELLIS_REPOSITORY_COMMIT: Final = "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
|
||||||
|
RELLIS_LABEL_CONFIG_SHA256: Final = (
|
||||||
|
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||||
|
)
|
||||||
|
RELLIS_EXAMPLE_POINTS_SHA256: Final = (
|
||||||
|
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||||
|
)
|
||||||
|
RELLIS_EXAMPLE_LABELS_SHA256: Final = (
|
||||||
|
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||||
|
)
|
||||||
|
RELLIS_EXAMPLE_FRAME_ID: Final = "000104"
|
||||||
|
RELLIS_LICENSE: Final = "CC-BY-NC-SA-3.0"
|
||||||
|
MAX_RELLIS_PREVIEW_POINTS: Final = 50_000
|
||||||
|
|
||||||
|
GroundTarget = Literal["ground", "non-ground", "ignore"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RellisClass:
|
||||||
|
label_id: int
|
||||||
|
class_name: str
|
||||||
|
bgr: tuple[int, int, int]
|
||||||
|
ground_target: GroundTarget
|
||||||
|
|
||||||
|
@property
|
||||||
|
def rgb(self) -> tuple[int, int, int]:
|
||||||
|
blue, green, red = self.bgr
|
||||||
|
return red, green, blue
|
||||||
|
|
||||||
|
@property
|
||||||
|
def hex(self) -> str:
|
||||||
|
red, green, blue = self.rgb
|
||||||
|
return f"#{red:02x}{green:02x}{blue:02x}"
|
||||||
|
|
||||||
|
def to_preview_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"label_id": self.label_id,
|
||||||
|
"class_name": self.class_name,
|
||||||
|
"hex": self.hex,
|
||||||
|
"ground_target": self.ground_target,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# IDs, names and BGR colors are pinned to the official repository config at
|
||||||
|
# RELLIS_REPOSITORY_COMMIT. The final column is Mission Core's versioned,
|
||||||
|
# algorithm-scoped ground evaluation policy; it is not an upstream claim.
|
||||||
|
RELLIS_CLASSES: Final[tuple[RellisClass, ...]] = (
|
||||||
|
RellisClass(0, "void", (0, 0, 0), "ignore"),
|
||||||
|
RellisClass(1, "dirt", (108, 64, 20), "ground"),
|
||||||
|
RellisClass(3, "grass", (0, 102, 0), "ground"),
|
||||||
|
RellisClass(4, "tree", (0, 255, 0), "non-ground"),
|
||||||
|
RellisClass(5, "pole", (0, 153, 153), "non-ground"),
|
||||||
|
RellisClass(6, "water", (0, 128, 255), "ignore"),
|
||||||
|
RellisClass(7, "sky", (0, 0, 255), "ignore"),
|
||||||
|
RellisClass(8, "vehicle", (255, 255, 0), "non-ground"),
|
||||||
|
RellisClass(9, "object", (255, 0, 127), "ignore"),
|
||||||
|
RellisClass(10, "asphalt", (64, 64, 64), "ground"),
|
||||||
|
RellisClass(12, "building", (255, 0, 0), "non-ground"),
|
||||||
|
RellisClass(15, "log", (102, 0, 0), "non-ground"),
|
||||||
|
RellisClass(17, "person", (204, 153, 255), "non-ground"),
|
||||||
|
RellisClass(18, "fence", (102, 0, 204), "non-ground"),
|
||||||
|
RellisClass(19, "bush", (255, 153, 204), "non-ground"),
|
||||||
|
RellisClass(23, "concrete", (170, 170, 170), "ground"),
|
||||||
|
RellisClass(27, "barrier", (41, 121, 255), "non-ground"),
|
||||||
|
RellisClass(31, "puddle", (134, 255, 239), "ignore"),
|
||||||
|
RellisClass(33, "mud", (99, 66, 34), "ground"),
|
||||||
|
RellisClass(34, "rubble", (110, 22, 138), "non-ground"),
|
||||||
|
)
|
||||||
|
RELLIS_CLASS_BY_ID: Final = {item.label_id: item for item in RELLIS_CLASSES}
|
||||||
|
|
||||||
|
|
||||||
|
class RellisSmokeError(RuntimeError):
|
||||||
|
"""The pinned official example cannot satisfy the RELLIS smoke contract."""
|
||||||
|
|
||||||
|
|
||||||
|
def build_rellis_official_smoke_preview(
|
||||||
|
point_path: Path,
|
||||||
|
label_path: Path,
|
||||||
|
output_path: Path,
|
||||||
|
*,
|
||||||
|
preview_points: int = 20_000,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Verify the official example and publish one bounded path-free preview."""
|
||||||
|
|
||||||
|
points_digest = _sha256_file(point_path)
|
||||||
|
labels_digest = _sha256_file(label_path)
|
||||||
|
if points_digest != RELLIS_EXAMPLE_POINTS_SHA256:
|
||||||
|
raise RellisSmokeError("RELLIS official example point digest is incompatible")
|
||||||
|
if labels_digest != RELLIS_EXAMPLE_LABELS_SHA256:
|
||||||
|
raise RellisSmokeError("RELLIS official example label digest is incompatible")
|
||||||
|
try:
|
||||||
|
frame = read_semantic_kitti_frame(point_path, label_path)
|
||||||
|
except DatasetFrameError as exc:
|
||||||
|
raise RellisSmokeError("RELLIS official example violates XYZI/label alignment") from exc
|
||||||
|
preview = rellis_native_scan_preview(
|
||||||
|
frame,
|
||||||
|
frame_id=RELLIS_EXAMPLE_FRAME_ID,
|
||||||
|
maximum_points=preview_points,
|
||||||
|
source_evidence={
|
||||||
|
"repository_url": RELLIS_REPOSITORY_URL,
|
||||||
|
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||||
|
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
|
||||||
|
"point_sha256": points_digest,
|
||||||
|
"label_sha256": labels_digest,
|
||||||
|
"license": RELLIS_LICENSE,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_atomic_json(output_path, preview)
|
||||||
|
return preview
|
||||||
|
|
||||||
|
|
||||||
|
def rellis_native_scan_preview(
|
||||||
|
frame: DatasetPointFrame,
|
||||||
|
*,
|
||||||
|
frame_id: str,
|
||||||
|
maximum_points: int,
|
||||||
|
source_evidence: dict[str, str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Build the common viewer artifact from a validated RELLIS native frame."""
|
||||||
|
|
||||||
|
if not frame_id or not 1 <= maximum_points <= MAX_RELLIS_PREVIEW_POINTS:
|
||||||
|
raise RellisSmokeError("RELLIS preview bounds are incompatible")
|
||||||
|
unknown = sorted(
|
||||||
|
int(value)
|
||||||
|
for value in np.unique(frame.semantic_labels)
|
||||||
|
if int(value) not in RELLIS_CLASS_BY_ID
|
||||||
|
)
|
||||||
|
if unknown:
|
||||||
|
raise RellisSmokeError("RELLIS frame contains labels absent from the pinned ontology")
|
||||||
|
required_evidence = {
|
||||||
|
"repository_url",
|
||||||
|
"repository_commit",
|
||||||
|
"label_config_sha256",
|
||||||
|
"point_sha256",
|
||||||
|
"label_sha256",
|
||||||
|
"license",
|
||||||
|
}
|
||||||
|
if set(source_evidence) != required_evidence or any(
|
||||||
|
not isinstance(value, str) or not value for value in source_evidence.values()
|
||||||
|
):
|
||||||
|
raise RellisSmokeError("RELLIS source evidence is incomplete")
|
||||||
|
|
||||||
|
sample_count = min(frame.point_count, maximum_points)
|
||||||
|
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
|
||||||
|
points = frame.points_xyz_m[indices]
|
||||||
|
remission = frame.remission[indices]
|
||||||
|
semantic = frame.semantic_labels[indices]
|
||||||
|
remission_min = float(np.min(remission))
|
||||||
|
remission_max = float(np.max(remission))
|
||||||
|
remission_span = remission_max - remission_min
|
||||||
|
if remission_span <= 0:
|
||||||
|
remission_u8 = np.zeros(sample_count, dtype=np.uint8)
|
||||||
|
else:
|
||||||
|
remission_u8 = np.rint(
|
||||||
|
(remission - remission_min) / remission_span * 255.0
|
||||||
|
).astype(np.uint8)
|
||||||
|
|
||||||
|
ground = np.zeros(sample_count, dtype=np.uint8)
|
||||||
|
evaluated = np.zeros(sample_count, dtype=np.uint8)
|
||||||
|
colors = np.zeros((sample_count, 3), dtype=np.uint8)
|
||||||
|
for label_id in np.unique(semantic):
|
||||||
|
item = RELLIS_CLASS_BY_ID[int(label_id)]
|
||||||
|
mask = semantic == label_id
|
||||||
|
colors[mask] = item.rgb
|
||||||
|
if item.ground_target != "ignore":
|
||||||
|
evaluated[mask] = 1
|
||||||
|
if item.ground_target == "ground":
|
||||||
|
ground[mask] = 1
|
||||||
|
|
||||||
|
present = Counter(int(value) for value in frame.semantic_labels)
|
||||||
|
present_classes = [
|
||||||
|
{
|
||||||
|
**RELLIS_CLASS_BY_ID[label_id].to_preview_dict(),
|
||||||
|
"source_point_count": count,
|
||||||
|
}
|
||||||
|
for label_id, count in sorted(present.items())
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"schema_version": RELLIS_PREVIEW_SCHEMA,
|
||||||
|
"source_id": RELLIS_SOURCE_ID,
|
||||||
|
"frame_id": frame_id,
|
||||||
|
"representation": "native-scan",
|
||||||
|
"sampling": "deterministic-even-index",
|
||||||
|
"source_point_count": frame.point_count,
|
||||||
|
"point_count": sample_count,
|
||||||
|
"points_xyz_m": points.tolist(),
|
||||||
|
"remission_0_to_255": remission_u8.tolist(),
|
||||||
|
"semantic_label_ids": semantic.astype(np.uint16).tolist(),
|
||||||
|
"semantic_rgb_0_to_255": colors.reshape(-1).tolist(),
|
||||||
|
"ground_truth_ground": ground.tolist(),
|
||||||
|
"evaluation_mask": evaluated.tolist(),
|
||||||
|
"classes": present_classes,
|
||||||
|
"coordinate_frame": {
|
||||||
|
"frame_id": "sensor/lidar/os1",
|
||||||
|
"handedness": "right",
|
||||||
|
"x": "forward",
|
||||||
|
"y": "left",
|
||||||
|
"z": "up",
|
||||||
|
"transform_applied": False,
|
||||||
|
},
|
||||||
|
"ground_policy": {
|
||||||
|
"schema_version": RELLIS_GROUND_POLICY_SCHEMA,
|
||||||
|
"ground": [
|
||||||
|
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ground"
|
||||||
|
],
|
||||||
|
"non_ground": [
|
||||||
|
item.class_name
|
||||||
|
for item in RELLIS_CLASSES
|
||||||
|
if item.ground_target == "non-ground"
|
||||||
|
],
|
||||||
|
"ignore": [
|
||||||
|
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ignore"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"source_evidence": source_evidence,
|
||||||
|
"safety": {
|
||||||
|
"visualization_only": True,
|
||||||
|
"compatibility_smoke_only": True,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
try:
|
||||||
|
with path.open("rb") as source:
|
||||||
|
for chunk in iter(lambda: source.read(1024**2), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
except OSError as exc:
|
||||||
|
raise RellisSmokeError("RELLIS official example is unavailable") from exc
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
mode="w",
|
||||||
|
dir=path.parent,
|
||||||
|
encoding="utf-8",
|
||||||
|
delete=False,
|
||||||
|
) as temporary:
|
||||||
|
temporary_path = Path(temporary.name)
|
||||||
|
json.dump(document, temporary, ensure_ascii=False, separators=(",", ":"))
|
||||||
|
temporary.flush()
|
||||||
|
os.fsync(temporary.fileno())
|
||||||
|
os.replace(temporary_path, path)
|
||||||
|
|
@ -426,6 +426,9 @@ app.include_router(
|
||||||
dataset_preview_provider=lambda: (
|
dataset_preview_provider=lambda: (
|
||||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
|
||||||
),
|
),
|
||||||
|
dataset_rellis_preview_provider=lambda: (
|
||||||
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
||||||
|
),
|
||||||
dataset_ground_preview_provider=lambda: (
|
dataset_ground_preview_provider=lambda: (
|
||||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import os
|
||||||
import re
|
import re
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Final
|
from typing import Annotated, Any, Final
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException, Query, Response
|
from fastapi import APIRouter, HTTPException, Query, Response
|
||||||
|
|
||||||
|
|
@ -21,6 +21,7 @@ from k1link.compute import (
|
||||||
lidar_pack_detail,
|
lidar_pack_detail,
|
||||||
)
|
)
|
||||||
from k1link.datasets import (
|
from k1link.datasets import (
|
||||||
|
RELLIS_SOURCE_ID,
|
||||||
DatasetAdmissionError,
|
DatasetAdmissionError,
|
||||||
configured_dataset_admission_manifest,
|
configured_dataset_admission_manifest,
|
||||||
configured_dataset_ground_preview,
|
configured_dataset_ground_preview,
|
||||||
|
|
@ -62,6 +63,7 @@ def build_lidar_router(
|
||||||
field_review_root_provider: RootProvider = configured_lidar_field_review_root,
|
field_review_root_provider: RootProvider = configured_lidar_field_review_root,
|
||||||
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
||||||
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
||||||
|
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
|
||||||
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||||
|
|
@ -70,18 +72,32 @@ def build_lidar_router(
|
||||||
def get_dataset_gateway() -> dict[str, object]:
|
def get_dataset_gateway() -> dict[str, object]:
|
||||||
return dataset_gateway_catalog(
|
return dataset_gateway_catalog(
|
||||||
admission_manifest_path=dataset_admission_provider(),
|
admission_manifest_path=dataset_admission_provider(),
|
||||||
|
rellis_preview_path=dataset_rellis_preview_provider(),
|
||||||
)
|
)
|
||||||
|
|
||||||
@router.get("/dataset-gateway/preview")
|
@router.get("/dataset-gateway/preview")
|
||||||
def get_dataset_gateway_preview() -> dict[str, Any]:
|
def get_dataset_gateway_preview(
|
||||||
path = dataset_preview_provider()
|
source_id: Annotated[
|
||||||
|
str,
|
||||||
|
Query(min_length=1, max_length=160),
|
||||||
|
] = "goose-3d/v2025-08-22",
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if source_id == "goose-3d/v2025-08-22":
|
||||||
|
path = dataset_preview_provider()
|
||||||
|
elif source_id == RELLIS_SOURCE_ID:
|
||||||
|
path = dataset_rellis_preview_provider()
|
||||||
|
else:
|
||||||
|
raise HTTPException(status_code=404, detail="Dataset source не найден.")
|
||||||
if path is None or not path.is_file():
|
if path is None or not path.is_file():
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=404,
|
status_code=404,
|
||||||
detail="Первый размеченный native scan ещё не импортирован.",
|
detail="Размеченный native scan источника ещё не импортирован.",
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
return read_dataset_native_scan_preview(path)
|
preview = read_dataset_native_scan_preview(path)
|
||||||
|
if preview["source_id"] != source_id:
|
||||||
|
raise DatasetAdmissionError("dataset preview source mismatch")
|
||||||
|
return preview
|
||||||
except DatasetAdmissionError as exc:
|
except DatasetAdmissionError as exc:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=500,
|
status_code=500,
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ from fastapi.routing import APIRoute
|
||||||
from k1link.datasets import (
|
from k1link.datasets import (
|
||||||
DatasetAdmissionError,
|
DatasetAdmissionError,
|
||||||
DatasetFrameError,
|
DatasetFrameError,
|
||||||
|
RELLIS_SOURCE_ID,
|
||||||
dataset_gateway_catalog,
|
dataset_gateway_catalog,
|
||||||
goose_admission,
|
goose_admission,
|
||||||
goose_benchmark,
|
goose_benchmark,
|
||||||
|
|
@ -19,6 +20,7 @@ from k1link.datasets import (
|
||||||
read_dataset_ground_preview,
|
read_dataset_ground_preview,
|
||||||
read_dataset_native_scan_preview,
|
read_dataset_native_scan_preview,
|
||||||
read_semantic_kitti_frame,
|
read_semantic_kitti_frame,
|
||||||
|
rellis_native_scan_preview,
|
||||||
)
|
)
|
||||||
from k1link.datasets.goose_admission import admit_goose_validation
|
from k1link.datasets.goose_admission import admit_goose_validation
|
||||||
from k1link.datasets.goose_benchmark import (
|
from k1link.datasets.goose_benchmark import (
|
||||||
|
|
@ -111,10 +113,14 @@ def test_dataset_gateway_api_is_read_only_and_path_free(
|
||||||
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
|
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
|
||||||
response = route() # type: ignore[operator]
|
response = route() # type: ignore[operator]
|
||||||
|
|
||||||
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2"
|
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v3"
|
||||||
assert response["access"] == "read-only"
|
assert response["access"] == "read-only"
|
||||||
assert response["storage"]["admitted"] is True
|
assert response["storage"]["admitted"] is True
|
||||||
assert response["storage"]["path_exposed"] is False
|
assert response["storage"]["path_exposed"] is False
|
||||||
|
assert [source["source_id"] for source in response["sources"]] == [
|
||||||
|
"goose-3d/v2025-08-22",
|
||||||
|
RELLIS_SOURCE_ID,
|
||||||
|
]
|
||||||
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
|
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
|
||||||
response["storage"]["required_wsl_root"], # type: ignore[index]
|
response["storage"]["required_wsl_root"], # type: ignore[index]
|
||||||
"",
|
"",
|
||||||
|
|
@ -286,6 +292,70 @@ def test_dataset_preview_api_is_read_only_and_integrity_checked(
|
||||||
assert response["frame_id"] == "frame-1"
|
assert response["frame_id"] == "frame-1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rellis_smoke_preview_keeps_semantickitti_alignment_and_ignore_policy(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
points_path = tmp_path / "000104.bin"
|
||||||
|
labels_path = tmp_path / "000104.label"
|
||||||
|
np.asarray(
|
||||||
|
[
|
||||||
|
[1.0, 0.0, -1.0, 0.1],
|
||||||
|
[2.0, 0.2, -0.8, 0.2],
|
||||||
|
[3.0, -0.2, 0.5, 0.3],
|
||||||
|
[4.0, 0.0, -0.5, 0.4],
|
||||||
|
],
|
||||||
|
dtype="<f4",
|
||||||
|
).tofile(points_path)
|
||||||
|
np.asarray([3, 23, 4, 31], dtype="<u4").tofile(labels_path)
|
||||||
|
frame = read_semantic_kitti_frame(points_path, labels_path)
|
||||||
|
|
||||||
|
preview = rellis_native_scan_preview(
|
||||||
|
frame,
|
||||||
|
frame_id="000104",
|
||||||
|
maximum_points=4,
|
||||||
|
source_evidence={
|
||||||
|
"repository_url": "https://github.com/unmannedlab/RELLIS-3D",
|
||||||
|
"repository_commit": "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
|
||||||
|
"label_config_sha256": (
|
||||||
|
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||||
|
),
|
||||||
|
"point_sha256": (
|
||||||
|
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||||
|
),
|
||||||
|
"label_sha256": (
|
||||||
|
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||||
|
),
|
||||||
|
"license": "CC-BY-NC-SA-3.0",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
preview_path = tmp_path / "rellis-preview.json"
|
||||||
|
preview_path.write_text(json.dumps(preview), encoding="utf-8")
|
||||||
|
|
||||||
|
decoded = read_dataset_native_scan_preview(preview_path)
|
||||||
|
|
||||||
|
assert decoded["source_id"] == RELLIS_SOURCE_ID
|
||||||
|
assert decoded["ground_truth_ground"] == [1, 1, 0, 0]
|
||||||
|
assert decoded["evaluation_mask"] == [1, 1, 1, 0]
|
||||||
|
assert decoded["coordinate_frame"]["x"] == "forward"
|
||||||
|
assert decoded["safety"]["compatibility_smoke_only"] is True
|
||||||
|
|
||||||
|
router = build_lidar_router(
|
||||||
|
dataset_admission_provider=lambda: None,
|
||||||
|
dataset_preview_provider=lambda: None,
|
||||||
|
dataset_rellis_preview_provider=lambda: preview_path,
|
||||||
|
)
|
||||||
|
route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview")
|
||||||
|
response = route(source_id=RELLIS_SOURCE_ID) # type: ignore[operator]
|
||||||
|
assert response["frame_id"] == "000104"
|
||||||
|
|
||||||
|
catalog = dataset_gateway_catalog(
|
||||||
|
Path("/mnt/d/NDC_MISSIONCORE/datasets"),
|
||||||
|
rellis_preview_path=preview_path,
|
||||||
|
)
|
||||||
|
rellis = catalog["sources"][1] # type: ignore[index]
|
||||||
|
assert rellis["admission"]["status"] == "smoke-ready" # type: ignore[index]
|
||||||
|
|
||||||
|
|
||||||
def test_goose_current_ground_is_scored_against_independent_labels(
|
def test_goose_current_ground_is_scored_against_independent_labels(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue