feat: add RELLIS dataset compatibility smoke
This commit is contained in:
parent
13c35ff446
commit
054feec0d2
|
|
@ -3,6 +3,57 @@ export type DatasetRepresentationId =
|
|||
| "normalized-scan"
|
||||
| "rolling-local-map";
|
||||
|
||||
export type DatasetSourceAdmissionStatus =
|
||||
| "blocked-storage-policy"
|
||||
| "ready-for-download"
|
||||
| "ready-for-smoke"
|
||||
| "downloading"
|
||||
| "downloaded"
|
||||
| "verifying"
|
||||
| "verified"
|
||||
| "frame-ready"
|
||||
| "smoke-ready";
|
||||
|
||||
export interface DatasetSource {
|
||||
sourceId: string;
|
||||
sourceKind: "goose" | "rellis";
|
||||
displayName: string;
|
||||
role: string;
|
||||
license: string;
|
||||
commercialUse:
|
||||
| "allowed-with-share-alike"
|
||||
| "research-only-license-review-required";
|
||||
format: string;
|
||||
frameSemantics: "one-lidar-revolution";
|
||||
sensor: string;
|
||||
platforms: string[];
|
||||
superclasses: string[];
|
||||
smokeExampleMb: number | null;
|
||||
primaryScanArchiveGb: number;
|
||||
labelArchiveGb: number | null;
|
||||
posesArchiveGb: number | null;
|
||||
fragmentCount: number;
|
||||
annotatedScanCount: number;
|
||||
admissionStatus: DatasetSourceAdmissionStatus;
|
||||
archive: {
|
||||
bytesTransferred: number;
|
||||
totalBytes: number;
|
||||
sizeBytes: number | null;
|
||||
sha256: string | null;
|
||||
integrity: string;
|
||||
vendorChecksumAvailable: false;
|
||||
} | null;
|
||||
frame: {
|
||||
frameId: string;
|
||||
pointCount: number;
|
||||
semanticClassCount: number;
|
||||
groundTruthGroundFraction: number;
|
||||
previewPointCount: number;
|
||||
previewSha256: string | null;
|
||||
previewAvailable: true;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface DatasetGatewayCatalog {
|
||||
storage: {
|
||||
configured: boolean;
|
||||
|
|
@ -13,42 +64,7 @@ export interface DatasetGatewayCatalog {
|
|||
requiredWindowsRoot: string;
|
||||
requiredWslRoot: string;
|
||||
};
|
||||
source: {
|
||||
sourceId: string;
|
||||
displayName: string;
|
||||
role: string;
|
||||
license: string;
|
||||
format: string;
|
||||
frameSemantics: "one-lidar-revolution";
|
||||
platforms: string[];
|
||||
superclasses: string[];
|
||||
validationArchiveGb: number;
|
||||
admissionStatus:
|
||||
| "blocked-storage-policy"
|
||||
| "ready-for-download"
|
||||
| "downloading"
|
||||
| "downloaded"
|
||||
| "verifying"
|
||||
| "verified"
|
||||
| "frame-ready";
|
||||
archive: {
|
||||
bytesTransferred: number;
|
||||
totalBytes: number;
|
||||
sizeBytes: number | null;
|
||||
sha256: string | null;
|
||||
integrity: string;
|
||||
vendorChecksumAvailable: false;
|
||||
} | null;
|
||||
frame: {
|
||||
frameId: string;
|
||||
pointCount: number;
|
||||
semanticClassCount: number;
|
||||
groundTruthGroundFraction: number;
|
||||
previewPointCount: number;
|
||||
previewSha256: string;
|
||||
previewAvailable: true;
|
||||
} | null;
|
||||
};
|
||||
sources: DatasetSource[];
|
||||
representations: Array<{
|
||||
id: DatasetRepresentationId;
|
||||
title: string;
|
||||
|
|
@ -72,7 +88,7 @@ export interface DatasetGatewayCatalog {
|
|||
}
|
||||
|
||||
export interface DatasetNativeScanPreview {
|
||||
sourceId: "goose-3d/v2025-08-22";
|
||||
sourceId: "goose-3d/v2025-08-22" | "rellis-3d/v1.1";
|
||||
frameId: string;
|
||||
sourcePointCount: number;
|
||||
pointCount: number;
|
||||
|
|
@ -81,13 +97,23 @@ export interface DatasetNativeScanPreview {
|
|||
semanticLabelIds: number[];
|
||||
semanticRgb0To255: number[];
|
||||
groundTruthGround: number[];
|
||||
evaluationMask: number[];
|
||||
classes: Array<{
|
||||
labelId: number;
|
||||
className: string;
|
||||
hex: string;
|
||||
challengeCategoryId: number;
|
||||
challengeCategoryName: string;
|
||||
challengeCategoryId: number | null;
|
||||
challengeCategoryName: string | null;
|
||||
groundTarget: "ground" | "non-ground" | "ignore" | null;
|
||||
sourcePointCount: number | null;
|
||||
}>;
|
||||
coordinateFrame: {
|
||||
frameId: string;
|
||||
x: "forward";
|
||||
y: "left";
|
||||
z: "up";
|
||||
} | null;
|
||||
compatibilitySmokeOnly: boolean;
|
||||
}
|
||||
|
||||
export interface DatasetGroundMetrics {
|
||||
|
|
@ -225,7 +251,7 @@ export function parseDatasetGatewayCatalog(
|
|||
): DatasetGatewayCatalog {
|
||||
const source = record(value, "Dataset Gateway");
|
||||
if (
|
||||
source.schema_version !== "missioncore.dataset-gateway-catalog/v2"
|
||||
source.schema_version !== "missioncore.dataset-gateway-catalog/v3"
|
||||
|| source.access !== "read-only"
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Dataset Gateway contract несовместим");
|
||||
|
|
@ -247,85 +273,158 @@ export function parseDatasetGatewayCatalog(
|
|||
throw new DatasetGatewayContractError("storage.attestation: неизвестное значение");
|
||||
}
|
||||
const sources = array(source.sources, "sources");
|
||||
if (sources.length !== 1) {
|
||||
throw new DatasetGatewayContractError("Ожидался один первичный dataset source");
|
||||
if (sources.length !== 2) {
|
||||
throw new DatasetGatewayContractError("Ожидались два закреплённых dataset source");
|
||||
}
|
||||
const dataset = record(sources[0], "sources[0]");
|
||||
const download = record(dataset.download, "source.download");
|
||||
const admission = record(dataset.admission, "source.admission");
|
||||
if (download.automatic !== false) {
|
||||
throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически");
|
||||
}
|
||||
const admissionStatus = admission.status;
|
||||
const parsedSources = sources.map((value, index): DatasetSource => {
|
||||
const dataset = record(value, `sources[${index}]`);
|
||||
const sourceKind = string(dataset.source_kind, "source_kind", true);
|
||||
const sourceId = string(dataset.source_id, "source_id", true);
|
||||
if (
|
||||
(sourceKind !== "goose" || sourceId !== "goose-3d/v2025-08-22")
|
||||
&& (sourceKind !== "rellis" || sourceId !== "rellis-3d/v1.1")
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Dataset source identity неизвестна");
|
||||
}
|
||||
if (dataset.frame_semantics !== "one-lidar-revolution") {
|
||||
throw new DatasetGatewayContractError("Dataset frame semantics несовместима");
|
||||
}
|
||||
const download = record(dataset.download, `sources[${index}].download`);
|
||||
const statistics = record(dataset.statistics, `sources[${index}].statistics`);
|
||||
const admission = record(dataset.admission, `sources[${index}].admission`);
|
||||
if (download.automatic !== false) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"Большой dataset нельзя загружать автоматически",
|
||||
);
|
||||
}
|
||||
const admissionStatus = admission.status;
|
||||
if (
|
||||
admissionStatus !== "ready-for-download"
|
||||
&& admissionStatus !== "ready-for-smoke"
|
||||
&& admissionStatus !== "blocked-storage-policy"
|
||||
&& admissionStatus !== "downloading"
|
||||
&& admissionStatus !== "downloaded"
|
||||
&& admissionStatus !== "verifying"
|
||||
&& admissionStatus !== "verified"
|
||||
&& admissionStatus !== "frame-ready"
|
||||
&& admissionStatus !== "smoke-ready"
|
||||
) {
|
||||
throw new DatasetGatewayContractError("source admission status неизвестен");
|
||||
}
|
||||
const archive = admission.archive === null
|
||||
? null
|
||||
: record(admission.archive, `sources[${index}].admission.archive`);
|
||||
const archiveValue = archive
|
||||
? {
|
||||
bytesTransferred: number(
|
||||
archive.bytes_transferred,
|
||||
"archive.bytes_transferred",
|
||||
),
|
||||
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
|
||||
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
|
||||
sha256: nullableDigest(archive.sha256, "archive.sha256"),
|
||||
integrity: string(archive.integrity, "archive.integrity", true),
|
||||
vendorChecksumAvailable: (() => {
|
||||
if (archive.vendor_checksum_available !== false) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"archive.vendor_checksum_available: ожидался false",
|
||||
);
|
||||
}
|
||||
return false as const;
|
||||
})(),
|
||||
}
|
||||
: null;
|
||||
const admittedFrame = admission.frame === null
|
||||
? null
|
||||
: record(admission.frame, `sources[${index}].admission.frame`);
|
||||
const frameValue = admittedFrame
|
||||
? {
|
||||
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
|
||||
pointCount: number(admittedFrame.point_count, "frame.point_count"),
|
||||
semanticClassCount: number(
|
||||
admittedFrame.semantic_class_count,
|
||||
"frame.semantic_class_count",
|
||||
),
|
||||
groundTruthGroundFraction: number(
|
||||
admittedFrame.ground_truth_ground_fraction,
|
||||
"frame.ground_truth_ground_fraction",
|
||||
),
|
||||
previewPointCount: number(
|
||||
admittedFrame.preview_point_count,
|
||||
"frame.preview_point_count",
|
||||
),
|
||||
previewSha256: nullableDigest(
|
||||
admittedFrame.preview_sha256,
|
||||
"frame.preview_sha256",
|
||||
),
|
||||
previewAvailable: (() => {
|
||||
if (admittedFrame.preview_available !== true) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"frame.preview_available: ожидался true",
|
||||
);
|
||||
}
|
||||
return true as const;
|
||||
})(),
|
||||
}
|
||||
: null;
|
||||
if (
|
||||
sourceKind === "goose"
|
||||
&& frameValue
|
||||
&& frameValue.previewSha256 === null
|
||||
) {
|
||||
throw new DatasetGatewayContractError("GOOSE frame.preview_sha256 отсутствует");
|
||||
}
|
||||
const commercialUse = dataset.commercial_use;
|
||||
if (
|
||||
commercialUse !== "allowed-with-share-alike"
|
||||
&& commercialUse !== "research-only-license-review-required"
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Dataset license scope неизвестен");
|
||||
}
|
||||
return {
|
||||
sourceId,
|
||||
sourceKind,
|
||||
displayName: string(dataset.display_name, "display_name"),
|
||||
role: string(dataset.role, "role", true),
|
||||
license: string(dataset.license, "license"),
|
||||
commercialUse,
|
||||
format: string(dataset.format, "format", true),
|
||||
frameSemantics: "one-lidar-revolution",
|
||||
sensor: string(dataset.sensor, "sensor"),
|
||||
platforms: displayStrings(dataset.platforms, "platforms"),
|
||||
superclasses: strings(dataset.superclasses, "superclasses"),
|
||||
smokeExampleMb: nullableNumber(
|
||||
download.smoke_example_mb,
|
||||
"smoke_example_mb",
|
||||
),
|
||||
primaryScanArchiveGb: number(
|
||||
download.primary_scan_archive_gb,
|
||||
"primary_scan_archive_gb",
|
||||
),
|
||||
labelArchiveGb: nullableNumber(
|
||||
download.label_archive_gb,
|
||||
"label_archive_gb",
|
||||
),
|
||||
posesArchiveGb: nullableNumber(
|
||||
download.poses_archive_gb,
|
||||
"poses_archive_gb",
|
||||
),
|
||||
fragmentCount: number(statistics.fragment_count, "fragment_count"),
|
||||
annotatedScanCount: number(
|
||||
statistics.annotated_scan_count,
|
||||
"annotated_scan_count",
|
||||
),
|
||||
admissionStatus,
|
||||
archive: archiveValue,
|
||||
frame: frameValue,
|
||||
};
|
||||
});
|
||||
if (
|
||||
admissionStatus !== "ready-for-download"
|
||||
&& admissionStatus !== "blocked-storage-policy"
|
||||
&& admissionStatus !== "downloading"
|
||||
&& admissionStatus !== "downloaded"
|
||||
&& admissionStatus !== "verifying"
|
||||
&& admissionStatus !== "verified"
|
||||
&& admissionStatus !== "frame-ready"
|
||||
parsedSources[0]?.sourceKind !== "goose"
|
||||
|| parsedSources[1]?.sourceKind !== "rellis"
|
||||
) {
|
||||
throw new DatasetGatewayContractError("source admission status неизвестен");
|
||||
throw new DatasetGatewayContractError("Dataset source order несовместим");
|
||||
}
|
||||
const archive = admission.archive === null
|
||||
? null
|
||||
: record(admission.archive, "source.admission.archive");
|
||||
const archiveValue = archive
|
||||
? {
|
||||
bytesTransferred: number(
|
||||
archive.bytes_transferred,
|
||||
"archive.bytes_transferred",
|
||||
),
|
||||
totalBytes: number(archive.total_bytes, "archive.total_bytes"),
|
||||
sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"),
|
||||
sha256: nullableDigest(archive.sha256, "archive.sha256"),
|
||||
integrity: string(archive.integrity, "archive.integrity", true),
|
||||
vendorChecksumAvailable: (() => {
|
||||
if (archive.vendor_checksum_available !== false) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"archive.vendor_checksum_available: ожидался false",
|
||||
);
|
||||
}
|
||||
return false as const;
|
||||
})(),
|
||||
}
|
||||
: null;
|
||||
const admittedFrame = admission.frame === null
|
||||
? null
|
||||
: record(admission.frame, "source.admission.frame");
|
||||
const frameValue = admittedFrame
|
||||
? {
|
||||
frameId: string(admittedFrame.frame_id, "frame.frame_id", true),
|
||||
pointCount: number(admittedFrame.point_count, "frame.point_count"),
|
||||
semanticClassCount: number(
|
||||
admittedFrame.semantic_class_count,
|
||||
"frame.semantic_class_count",
|
||||
),
|
||||
groundTruthGroundFraction: number(
|
||||
admittedFrame.ground_truth_ground_fraction,
|
||||
"frame.ground_truth_ground_fraction",
|
||||
),
|
||||
previewPointCount: number(
|
||||
admittedFrame.preview_point_count,
|
||||
"frame.preview_point_count",
|
||||
),
|
||||
previewSha256: nullableDigest(
|
||||
admittedFrame.preview_sha256,
|
||||
"frame.preview_sha256",
|
||||
) ?? (() => {
|
||||
throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует");
|
||||
})(),
|
||||
previewAvailable: (() => {
|
||||
if (admittedFrame.preview_available !== true) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"frame.preview_available: ожидался true",
|
||||
);
|
||||
}
|
||||
return true as const;
|
||||
})(),
|
||||
}
|
||||
: null;
|
||||
const representations = array(
|
||||
source.representations,
|
||||
"representations",
|
||||
|
|
@ -366,9 +465,6 @@ export function parseDatasetGatewayCatalog(
|
|||
) {
|
||||
throw new DatasetGatewayContractError("Vendor-map boundary завышен");
|
||||
}
|
||||
if (dataset.frame_semantics !== "one-lidar-revolution") {
|
||||
throw new DatasetGatewayContractError("GOOSE frame semantics несовместима");
|
||||
}
|
||||
return {
|
||||
storage: {
|
||||
configured: boolean(storage.configured, "storage.configured"),
|
||||
|
|
@ -382,23 +478,7 @@ export function parseDatasetGatewayCatalog(
|
|||
),
|
||||
requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"),
|
||||
},
|
||||
source: {
|
||||
sourceId: string(dataset.source_id, "source_id", true),
|
||||
displayName: string(dataset.display_name, "display_name"),
|
||||
role: string(dataset.role, "role", true),
|
||||
license: string(dataset.license, "license"),
|
||||
format: string(dataset.format, "format", true),
|
||||
frameSemantics: "one-lidar-revolution",
|
||||
platforms: displayStrings(dataset.platforms, "platforms"),
|
||||
superclasses: strings(dataset.superclasses, "superclasses"),
|
||||
validationArchiveGb: number(
|
||||
download.validation_archive_gb,
|
||||
"validation_archive_gb",
|
||||
),
|
||||
admissionStatus,
|
||||
archive: archiveValue,
|
||||
frame: frameValue,
|
||||
},
|
||||
sources: parsedSources,
|
||||
representations,
|
||||
pipeline,
|
||||
currentInput: {
|
||||
|
|
@ -417,9 +497,12 @@ export function parseDatasetNativeScanPreview(
|
|||
value: unknown,
|
||||
): DatasetNativeScanPreview {
|
||||
const source = record(value, "Dataset preview");
|
||||
const isGoose = source.schema_version === "missioncore.dataset-native-scan-preview/v1"
|
||||
&& source.source_id === "goose-3d/v2025-08-22";
|
||||
const isRellis = source.schema_version === "missioncore.dataset-native-scan-preview/v2"
|
||||
&& source.source_id === "rellis-3d/v1.1";
|
||||
if (
|
||||
source.schema_version !== "missioncore.dataset-native-scan-preview/v1"
|
||||
|| source.source_id !== "goose-3d/v2025-08-22"
|
||||
(!isGoose && !isRellis)
|
||||
|| source.representation !== "native-scan"
|
||||
|| source.sampling !== "deterministic-even-index"
|
||||
) {
|
||||
|
|
@ -463,6 +546,9 @@ export function parseDatasetNativeScanPreview(
|
|||
"ground_truth_ground",
|
||||
1,
|
||||
);
|
||||
const evaluationMask = isRellis
|
||||
? integers(source.evaluation_mask, "evaluation_mask", 1)
|
||||
: Array.from({ length: pointCount }, () => 1);
|
||||
if (
|
||||
pointCount < 1
|
||||
|| pointCount > 50_000
|
||||
|
|
@ -472,35 +558,74 @@ export function parseDatasetNativeScanPreview(
|
|||
|| semanticLabelIds.length !== pointCount
|
||||
|| semanticRgb0To255.length !== pointCount * 3
|
||||
|| groundTruthGround.length !== pointCount
|
||||
|| evaluationMask.length !== pointCount
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Dataset preview arrays не выровнены");
|
||||
}
|
||||
const classes = array(source.classes, "classes").map((value, index) => {
|
||||
const item = record(value, `classes[${index}]`);
|
||||
const groundTarget = item.ground_target;
|
||||
if (
|
||||
groundTarget !== undefined
|
||||
&& groundTarget !== "ground"
|
||||
&& groundTarget !== "non-ground"
|
||||
&& groundTarget !== "ignore"
|
||||
) {
|
||||
throw new DatasetGatewayContractError("class.ground_target неизвестен");
|
||||
}
|
||||
const normalizedGroundTarget: "ground" | "non-ground" | "ignore" | null =
|
||||
groundTarget === undefined ? null : groundTarget;
|
||||
return {
|
||||
labelId: number(item.label_id, "class.label_id"),
|
||||
className: string(item.class_name, "class.class_name", true),
|
||||
hex: string(item.hex, "class.hex"),
|
||||
challengeCategoryId: number(
|
||||
item.challenge_category_id,
|
||||
"class.challenge_category_id",
|
||||
),
|
||||
challengeCategoryName: string(
|
||||
item.challenge_category_name,
|
||||
"class.challenge_category_name",
|
||||
true,
|
||||
),
|
||||
challengeCategoryId: item.challenge_category_id === undefined
|
||||
? null
|
||||
: number(item.challenge_category_id, "class.challenge_category_id"),
|
||||
challengeCategoryName: item.challenge_category_name === undefined
|
||||
? null
|
||||
: string(
|
||||
item.challenge_category_name,
|
||||
"class.challenge_category_name",
|
||||
true,
|
||||
),
|
||||
groundTarget: normalizedGroundTarget,
|
||||
sourcePointCount: item.source_point_count === undefined
|
||||
? null
|
||||
: number(item.source_point_count, "class.source_point_count"),
|
||||
};
|
||||
});
|
||||
const safety = record(source.safety, "safety");
|
||||
if (
|
||||
safety.visualization_only !== true
|
||||
|| safety.navigation_or_safety_accepted !== false
|
||||
|| (isRellis && safety.compatibility_smoke_only !== true)
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен");
|
||||
}
|
||||
const coordinateFrame = isRellis
|
||||
? (() => {
|
||||
const frame = record(source.coordinate_frame, "coordinate_frame");
|
||||
if (
|
||||
frame.frame_id !== "sensor/lidar/os1"
|
||||
|| frame.x !== "forward"
|
||||
|| frame.y !== "left"
|
||||
|| frame.z !== "up"
|
||||
|| frame.handedness !== "right"
|
||||
|| frame.transform_applied !== false
|
||||
) {
|
||||
throw new DatasetGatewayContractError("RELLIS coordinate frame несовместим");
|
||||
}
|
||||
return {
|
||||
frameId: "sensor/lidar/os1",
|
||||
x: "forward" as const,
|
||||
y: "left" as const,
|
||||
z: "up" as const,
|
||||
};
|
||||
})()
|
||||
: null;
|
||||
return {
|
||||
sourceId: "goose-3d/v2025-08-22",
|
||||
sourceId: isRellis ? "rellis-3d/v1.1" : "goose-3d/v2025-08-22",
|
||||
frameId: string(source.frame_id, "frame_id", true),
|
||||
sourcePointCount,
|
||||
pointCount,
|
||||
|
|
@ -509,7 +634,10 @@ export function parseDatasetNativeScanPreview(
|
|||
semanticLabelIds,
|
||||
semanticRgb0To255,
|
||||
groundTruthGround,
|
||||
evaluationMask,
|
||||
classes,
|
||||
coordinateFrame,
|
||||
compatibilitySmokeOnly: isRellis,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -711,14 +839,22 @@ export async function fetchDatasetGatewayCatalog(
|
|||
}
|
||||
|
||||
export async function fetchDatasetNativeScanPreview(
|
||||
options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {},
|
||||
options: {
|
||||
sourceId?: DatasetNativeScanPreview["sourceId"];
|
||||
signal?: AbortSignal;
|
||||
fetcher?: DatasetFetch;
|
||||
} = {},
|
||||
): Promise<DatasetNativeScanPreview> {
|
||||
const fetcher = options.fetcher ?? fetch;
|
||||
const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", {
|
||||
const sourceId = options.sourceId ?? "goose-3d/v2025-08-22";
|
||||
const response = await fetcher(
|
||||
`/api/v1/lidar/dataset-gateway/preview?source_id=${encodeURIComponent(sourceId)}`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
signal: options.signal,
|
||||
});
|
||||
},
|
||||
);
|
||||
return parseDatasetNativeScanPreview(await responseJson(response));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -224,7 +224,9 @@
|
|||
}
|
||||
|
||||
.dataset-fragment-comparison,
|
||||
.dataset-fragment-risks {
|
||||
.dataset-fragment-risks,
|
||||
.rellis-review__metrics,
|
||||
.rellis-review__policy > div {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1223,6 +1223,85 @@
|
|||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.rellis-review__metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.rellis-review__metrics article {
|
||||
display: grid;
|
||||
gap: 0.2rem;
|
||||
border-radius: 0.85rem;
|
||||
background: var(--station-panel);
|
||||
padding: 0.78rem 0.85rem;
|
||||
}
|
||||
|
||||
.rellis-review__metrics span,
|
||||
.rellis-review__metrics small {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.55rem;
|
||||
}
|
||||
|
||||
.rellis-review__metrics strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 1.05rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy {
|
||||
display: grid;
|
||||
gap: 0.7rem;
|
||||
border-radius: 1rem;
|
||||
background: var(--station-panel);
|
||||
padding: 0.9rem 1rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy h3,
|
||||
.rellis-review__policy p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.rellis-review__policy h3 {
|
||||
margin-top: 0.28rem;
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy p {
|
||||
max-width: 52rem;
|
||||
margin-top: 0.3rem;
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.6rem;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.rellis-review__policy > div {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy article {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 0.35rem;
|
||||
border-radius: 0.75rem;
|
||||
background: rgb(255 255 255 / 0.03);
|
||||
padding: 0.7rem 0.75rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy article span {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.54rem;
|
||||
}
|
||||
|
||||
.rellis-review__policy article strong {
|
||||
color: var(--nodedc-text-secondary);
|
||||
font-size: 0.62rem;
|
||||
font-weight: 560;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.dataset-sequence-timeline {
|
||||
height: 2rem;
|
||||
}
|
||||
|
|
@ -2043,6 +2122,11 @@
|
|||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.dataset-library__entries {
|
||||
display: grid;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.dataset-entry {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(20rem, 1.2fr) minmax(22rem, 0.8fr);
|
||||
|
|
|
|||
|
|
@ -8,12 +8,15 @@ import {
|
|||
import {
|
||||
fetchDatasetGatewayCatalog,
|
||||
type DatasetGatewayCatalog,
|
||||
type DatasetSource,
|
||||
type DatasetSourceAdmissionStatus,
|
||||
} from "../core/lidar/datasetGateway";
|
||||
import {
|
||||
fetchPolygonRunCatalog,
|
||||
type PolygonRunCatalog,
|
||||
} from "../core/polygon/runArchive";
|
||||
import { GooseDatasetReview } from "./GooseDatasetReview";
|
||||
import { RellisDatasetReview } from "./RellisDatasetReview";
|
||||
|
||||
const representationLabels: Record<string, string> = {
|
||||
"native-scan": "Исходный скан",
|
||||
|
|
@ -27,15 +30,17 @@ function errorMessage(error: unknown): string {
|
|||
: "Каталог датасетов недоступен.";
|
||||
}
|
||||
|
||||
function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) {
|
||||
const labels: Record<typeof status, string> = {
|
||||
function admissionLabel(status: DatasetSourceAdmissionStatus): string {
|
||||
const labels: Record<DatasetSourceAdmissionStatus, string> = {
|
||||
"blocked-storage-policy": "Worker не настроен",
|
||||
"ready-for-download": "Готов к загрузке",
|
||||
"ready-for-smoke": "Готов к smoke",
|
||||
downloading: "Загружается",
|
||||
downloaded: "Загружен",
|
||||
verifying: "Проверяется",
|
||||
verified: "Проверен",
|
||||
"frame-ready": "Первый кадр готов",
|
||||
"frame-ready": "Dataset готов",
|
||||
"smoke-ready": "Smoke готов",
|
||||
};
|
||||
return labels[status];
|
||||
}
|
||||
|
|
@ -55,12 +60,42 @@ function isGooseQualificationRun(
|
|||
&& item.scenarioGeneration.startsWith("goose-3d");
|
||||
}
|
||||
|
||||
function sourceReady(source: DatasetSource): boolean {
|
||||
return source.admissionStatus === "frame-ready"
|
||||
|| source.admissionStatus === "smoke-ready";
|
||||
}
|
||||
|
||||
function sourceDescription(source: DatasetSource): string {
|
||||
if (source.sourceKind === "goose") {
|
||||
return "Восемь фрагментов и 961 разреженный размеченный оборот VLS-128. Здесь уже опубликовано покадровое сравнение текущего ground baseline и Patchwork++.";
|
||||
}
|
||||
return "Независимый off-road источник с Ouster OS1 64. Сейчас открыт официальный frame 000104 для проверки reader, осей, ontology и ground-policy; полный benchmark ещё не выполнялся.";
|
||||
}
|
||||
|
||||
function sourceFooter(source: DatasetSource, storageReady: boolean): string {
|
||||
if (source.admissionStatus === "frame-ready") {
|
||||
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
|
||||
}
|
||||
if (source.admissionStatus === "smoke-ready") {
|
||||
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
|
||||
}
|
||||
if (source.admissionStatus === "downloading") {
|
||||
return "Архив остаётся на D worker; после загрузки проверяются digest, лицензия и point-label alignment.";
|
||||
}
|
||||
if (!storageReady) {
|
||||
return "Сначала нужен допущенный Dataset Root на диске D worker.";
|
||||
}
|
||||
return source.sourceKind === "rellis"
|
||||
? "Следующий шаг: пропустить официальный 24 МБ example через smoke admission."
|
||||
: "Следующий шаг: загрузить validation archive на worker и проверить hash/license.";
|
||||
}
|
||||
|
||||
export function DatasetGatewayWorkspace() {
|
||||
const [catalog, setCatalog] = useState<DatasetGatewayCatalog | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [reloadGeneration, setReloadGeneration] = useState(0);
|
||||
const [previewOpen, setPreviewOpen] = useState(false);
|
||||
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
|
||||
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
|
||||
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
|
||||
const [runError, setRunError] = useState<string | null>(null);
|
||||
|
|
@ -85,7 +120,9 @@ export function DatasetGatewayWorkspace() {
|
|||
}, [reloadGeneration]);
|
||||
|
||||
useEffect(() => {
|
||||
if (catalog?.source.admissionStatus !== "downloading") return;
|
||||
if (!catalog?.sources.some((source) => source.admissionStatus === "downloading")) {
|
||||
return;
|
||||
}
|
||||
const timer = window.setTimeout(
|
||||
() => setReloadGeneration((value) => value + 1),
|
||||
5_000,
|
||||
|
|
@ -113,10 +150,10 @@ export function DatasetGatewayWorkspace() {
|
|||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const downloadProgress = catalog?.source.archive
|
||||
? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes
|
||||
: null;
|
||||
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
|
||||
const openSource = catalog?.sources.find(
|
||||
(source) => source.sourceId === openSourceId,
|
||||
) ?? null;
|
||||
|
||||
return (
|
||||
<div className="standard-workspace dataset-workspace">
|
||||
|
|
@ -125,17 +162,17 @@ export function DatasetGatewayWorkspace() {
|
|||
<span className="section-eyebrow">ЗАЧЕМ ЭТО НУЖНО</span>
|
||||
<h2>Независимая проверка алгоритмов</h2>
|
||||
<p>
|
||||
Датасет содержит известные размеченные данные. Мы показываем
|
||||
конкретный фрагмент, результаты каждого алгоритма и ошибки
|
||||
относительно разметки. Он не подмешивается в диагностику реального
|
||||
сенсора и не заменяет closed-loop симуляцию.
|
||||
Каждый источник остаётся отдельным: собственные sensor domain,
|
||||
разметка, лицензия и результаты. Совпадение результата на GOOSE и
|
||||
RELLIS снижает риск подгонки под один датасет, но не заменяет replay
|
||||
реального сенсора и closed-loop симуляцию.
|
||||
</p>
|
||||
</div>
|
||||
<ol aria-label="Как использовать датасет">
|
||||
<li><span>01</span>Выбрать фрагмент</li>
|
||||
<li><span>02</span>Посмотреть сканы</li>
|
||||
<li><span>01</span>Выбрать источник</li>
|
||||
<li><span>02</span>Проверить scan и labels</li>
|
||||
<li><span>03</span>Сравнить алгоритмы</li>
|
||||
<li><span>04</span>Проверить риски</li>
|
||||
<li><span>04</span>Проверить переносимость</li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
|
|
@ -171,80 +208,91 @@ export function DatasetGatewayWorkspace() {
|
|||
</StatusBadge>
|
||||
</header>
|
||||
|
||||
<article className="dataset-entry">
|
||||
<div className="dataset-entry__identity">
|
||||
<span className="dataset-entry__mark">G</span>
|
||||
<div>
|
||||
<span>PUBLIC · OFF-ROAD · LABELED</span>
|
||||
<h3>{catalog.source.displayName}</h3>
|
||||
<p>
|
||||
Validation split: восемь фрагментов исходных записей и 961
|
||||
разреженный размеченный оборот VLS-128. Это коллекция native
|
||||
scans для perception, а не непрерывное видео или траектория
|
||||
машины.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<dl>
|
||||
<div>
|
||||
<dt>Состояние</dt>
|
||||
<dd>{admissionLabel(catalog.source.admissionStatus)}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Validation</dt>
|
||||
<dd>{catalog.source.validationArchiveGb} ГБ</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Группы</dt>
|
||||
<dd>{catalog.source.superclasses.length} superclass</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Лицензия</dt>
|
||||
<dd>{catalog.source.license}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
{catalog.source.admissionStatus === "downloading"
|
||||
&& catalog.source.archive
|
||||
&& downloadProgress !== null ? (
|
||||
<div className="dataset-download" aria-label="Загрузка GOOSE validation">
|
||||
<span style={{ width: `${downloadProgress * 100}%` }} />
|
||||
<p>
|
||||
{formatBytes(catalog.source.archive.bytesTransferred)}
|
||||
{" из "}
|
||||
{formatBytes(catalog.source.archive.totalBytes)}
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
<footer>
|
||||
<p>
|
||||
{catalog.source.admissionStatus === "frame-ready"
|
||||
? "Native scan и разметка прошли admission. Можно открыть независимый ground truth."
|
||||
: catalog.source.admissionStatus === "downloading"
|
||||
? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment."
|
||||
: catalog.storage.admitted
|
||||
? "Следующий шаг: загрузить validation archive на worker и проверить hash/license."
|
||||
: "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}
|
||||
</p>
|
||||
<div>
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
disabled={catalog.source.admissionStatus !== "frame-ready"}
|
||||
title={
|
||||
catalog.source.admissionStatus === "frame-ready"
|
||||
? "Открыть фрагменты, сканы и результаты анализа"
|
||||
: "Размеченные сканы ещё не импортированы"
|
||||
}
|
||||
onClick={() => setPreviewOpen((value) => !value)}
|
||||
<div className="dataset-library__entries">
|
||||
{catalog.sources.map((source) => {
|
||||
const downloadProgress = source.archive
|
||||
? source.archive.bytesTransferred / source.archive.totalBytes
|
||||
: null;
|
||||
const opened = source.sourceId === openSourceId;
|
||||
return (
|
||||
<article
|
||||
className="dataset-entry"
|
||||
data-source={source.sourceKind}
|
||||
key={source.sourceId}
|
||||
>
|
||||
{previewOpen ? "Закрыть датасет" : "Открыть датасет"}
|
||||
</Button>
|
||||
</div>
|
||||
</footer>
|
||||
</article>
|
||||
<div className="dataset-entry__identity">
|
||||
<span className="dataset-entry__mark">
|
||||
{source.sourceKind === "goose" ? "G" : "R"}
|
||||
</span>
|
||||
<div>
|
||||
<span>
|
||||
PUBLIC · OFF-ROAD · LABELED
|
||||
{source.sourceKind === "rellis" ? " · RESEARCH" : ""}
|
||||
</span>
|
||||
<h3>{source.displayName}</h3>
|
||||
<p>{sourceDescription(source)}</p>
|
||||
</div>
|
||||
</div>
|
||||
<dl>
|
||||
<div>
|
||||
<dt>Состояние</dt>
|
||||
<dd>{admissionLabel(source.admissionStatus)}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Сканер</dt>
|
||||
<dd>{source.sensor}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Размеченных сканов</dt>
|
||||
<dd>{source.annotatedScanCount.toLocaleString("ru-RU")}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Лицензия</dt>
|
||||
<dd>{source.license}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
{source.admissionStatus === "downloading"
|
||||
&& source.archive
|
||||
&& downloadProgress !== null ? (
|
||||
<div
|
||||
className="dataset-download"
|
||||
aria-label={`Загрузка ${source.displayName}`}
|
||||
>
|
||||
<span style={{ width: `${downloadProgress * 100}%` }} />
|
||||
<p>
|
||||
{formatBytes(source.archive.bytesTransferred)}
|
||||
{" из "}
|
||||
{formatBytes(source.archive.totalBytes)}
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
<footer>
|
||||
<p>{sourceFooter(source, catalog.storage.admitted)}</p>
|
||||
<div>
|
||||
<Button
|
||||
size="compact"
|
||||
variant="secondary"
|
||||
disabled={!sourceReady(source)}
|
||||
title={
|
||||
sourceReady(source)
|
||||
? "Открыть содержимое источника"
|
||||
: "Совместимый preview ещё не опубликован"
|
||||
}
|
||||
onClick={() => setOpenSourceId(
|
||||
opened ? null : source.sourceId,
|
||||
)}
|
||||
>
|
||||
{opened ? "Закрыть датасет" : "Открыть датасет"}
|
||||
</Button>
|
||||
</div>
|
||||
</footer>
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</GlassSurface>
|
||||
|
||||
{previewOpen && selectedRunId ? (
|
||||
{openSource?.sourceKind === "goose" && selectedRunId ? (
|
||||
<>
|
||||
{gooseRuns.length > 1 ? (
|
||||
<div className="dataset-result-select">
|
||||
|
|
@ -267,12 +315,14 @@ export function DatasetGatewayWorkspace() {
|
|||
) : null}
|
||||
<GooseDatasetReview runId={selectedRunId} />
|
||||
</>
|
||||
) : previewOpen ? (
|
||||
) : openSource?.sourceKind === "goose" ? (
|
||||
<section className="polygon-review-unavailable">
|
||||
<StatusBadge tone="warning">Review недоступен</StatusBadge>
|
||||
<h3>Датасет принят, но результаты анализа не опубликованы</h3>
|
||||
<h3>GOOSE принят, но анализ не опубликован</h3>
|
||||
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
|
||||
</section>
|
||||
) : openSource?.sourceKind === "rellis" ? (
|
||||
<RellisDatasetReview />
|
||||
) : null}
|
||||
|
||||
<details className="dataset-contract">
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ export interface LidarGroundPointCloudFrame {
|
|||
disagreement: number[];
|
||||
candidateDisagreement?: number[];
|
||||
groundTruthGround?: number[];
|
||||
evaluationMask?: number[];
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -87,13 +88,18 @@ function frameColors(
|
|||
);
|
||||
} else if (mode === "ground-truth") {
|
||||
const groundTruth = frame.masks.groundTruthGround?.[index] === 1;
|
||||
setRgb(
|
||||
colors,
|
||||
offset,
|
||||
groundTruth ? 0.76 : 0.29,
|
||||
groundTruth ? 0.86 : 0.33,
|
||||
groundTruth ? 0.52 : 0.36,
|
||||
);
|
||||
const evaluated = frame.masks.evaluationMask?.[index] !== 0;
|
||||
if (!evaluated) {
|
||||
setRgb(colors, offset, 0.86, 0.61, 0.24);
|
||||
} else {
|
||||
setRgb(
|
||||
colors,
|
||||
offset,
|
||||
groundTruth ? 0.76 : 0.29,
|
||||
groundTruth ? 0.86 : 0.33,
|
||||
groundTruth ? 0.52 : 0.36,
|
||||
);
|
||||
}
|
||||
} else if (mode === "current") {
|
||||
setRgb(
|
||||
colors,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,239 @@
|
|||
import { useEffect, useMemo, useState } from "react";
|
||||
import { StatusBadge } from "@nodedc/ui-react";
|
||||
|
||||
import {
|
||||
fetchDatasetNativeScanPreview,
|
||||
type DatasetNativeScanPreview,
|
||||
} from "../core/lidar/datasetGateway";
|
||||
import {
|
||||
LidarGroundPointCloud,
|
||||
type LidarGroundViewMode,
|
||||
} from "./LidarGroundPointCloud";
|
||||
|
||||
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
|
||||
["intensity", "Исходный скан"],
|
||||
["semantic", "Классы RELLIS"],
|
||||
["ground-truth", "Ground target"],
|
||||
];
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return new Intl.NumberFormat("ru-RU", {
|
||||
style: "percent",
|
||||
maximumFractionDigits: 1,
|
||||
}).format(value);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error && error.message.trim()
|
||||
? error.message
|
||||
: "Официальный пример RELLIS недоступен.";
|
||||
}
|
||||
|
||||
function modeExplanation(mode: LidarGroundViewMode): string {
|
||||
if (mode === "semantic") {
|
||||
return "Цвета из закреплённой официальной ontology RELLIS-3D.";
|
||||
}
|
||||
if (mode === "ground-truth") {
|
||||
return "Зелёный — ground, серый — non-ground, янтарный — ignore.";
|
||||
}
|
||||
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
|
||||
}
|
||||
|
||||
export function RellisDatasetReview() {
|
||||
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
|
||||
|
||||
useEffect(() => {
|
||||
const controller = new AbortController();
|
||||
setError(null);
|
||||
void fetchDatasetNativeScanPreview({
|
||||
sourceId: "rellis-3d/v1.1",
|
||||
signal: controller.signal,
|
||||
})
|
||||
.then((value) => {
|
||||
if (!controller.signal.aborted) setPreview(value);
|
||||
})
|
||||
.catch((loadError: unknown) => {
|
||||
if (!controller.signal.aborted) setError(errorMessage(loadError));
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, []);
|
||||
|
||||
const scene = useMemo(() => {
|
||||
if (!preview) return null;
|
||||
const empty = Array.from({ length: preview.pointCount }, () => 0);
|
||||
return {
|
||||
pointCount: preview.pointCount,
|
||||
pointsXyzM: preview.pointsXyzM,
|
||||
intensity0To255: preview.remission0To255,
|
||||
semanticRgb0To255: preview.semanticRgb0To255,
|
||||
masks: {
|
||||
currentGround: empty,
|
||||
currentAssigned: preview.evaluationMask,
|
||||
candidateGround: empty,
|
||||
candidateAssigned: preview.evaluationMask,
|
||||
disagreement: empty,
|
||||
groundTruthGround: preview.groundTruthGround,
|
||||
evaluationMask: preview.evaluationMask,
|
||||
},
|
||||
};
|
||||
}, [preview]);
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<section className="polygon-review-unavailable">
|
||||
<StatusBadge tone="danger">Smoke preview недоступен</StatusBadge>
|
||||
<h3>RELLIS-3D не открыт</h3>
|
||||
<p>{error}</p>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
if (!preview || !scene) {
|
||||
return (
|
||||
<section className="polygon-review-unavailable">
|
||||
<StatusBadge tone="accent">Проверяем пример</StatusBadge>
|
||||
<h3>Открываем официальный RELLIS scan</h3>
|
||||
<p>Исходные dataset bytes не переносятся в браузер.</p>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
const evaluatedPoints = preview.evaluationMask.reduce(
|
||||
(total, value) => total + value,
|
||||
0,
|
||||
);
|
||||
const groundPoints = preview.groundTruthGround.reduce(
|
||||
(total, value, index) => total + value * (preview.evaluationMask[index] ?? 0),
|
||||
0,
|
||||
);
|
||||
const groundClasses = preview.classes.filter(
|
||||
(item) => item.groundTarget === "ground",
|
||||
);
|
||||
const nonGroundClasses = preview.classes.filter(
|
||||
(item) => item.groundTarget === "non-ground",
|
||||
);
|
||||
const ignoredClasses = preview.classes.filter(
|
||||
(item) => item.groundTarget === "ignore",
|
||||
);
|
||||
|
||||
return (
|
||||
<section
|
||||
className="dataset-sequence-review rellis-review"
|
||||
aria-label="Просмотр официального примера RELLIS-3D"
|
||||
>
|
||||
<header className="dataset-sequence-review__truth">
|
||||
<div>
|
||||
<span className="section-eyebrow">RELLIS S0 · COMPATIBILITY SMOKE</span>
|
||||
<h2>Официальный Ouster scan читается без конвертации формата</h2>
|
||||
<p>
|
||||
Frame {preview.frameId}: исходные XYZI и point-wise labels остались
|
||||
выровнены. Это проверка reader, осей, цветов и ground-policy — ещё не
|
||||
сравнение алгоритмов и не основание для навигации.
|
||||
</p>
|
||||
</div>
|
||||
<dl>
|
||||
<div>
|
||||
<dt>Исходных точек</dt>
|
||||
<dd>{preview.sourcePointCount.toLocaleString("ru-RU")}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Browser preview</dt>
|
||||
<dd>{preview.pointCount.toLocaleString("ru-RU")}</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Sensor frame</dt>
|
||||
<dd>{preview.coordinateFrame?.frameId ?? "—"}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</header>
|
||||
|
||||
<section className="polygon-review-player">
|
||||
<header className="polygon-review-player__header">
|
||||
<div>
|
||||
<span className="section-eyebrow">OFFICIAL EXAMPLE · FRAME 000104</span>
|
||||
<strong>Ouster OS1 64 · X forward · Y left · Z up</strong>
|
||||
</div>
|
||||
<div className="polygon-review-mode" aria-label="Режим RELLIS preview">
|
||||
{modes.map(([viewMode, label]) => (
|
||||
<button
|
||||
type="button"
|
||||
key={viewMode}
|
||||
data-active={mode === viewMode ? "true" : undefined}
|
||||
onClick={() => setMode(viewMode)}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</header>
|
||||
<div className="polygon-review-stage">
|
||||
<LidarGroundPointCloud
|
||||
frame={scene}
|
||||
mode={mode}
|
||||
coordinateFrame="sensor-fixed"
|
||||
/>
|
||||
<div className="polygon-review-stage__legend">
|
||||
<strong>{modes.find(([viewMode]) => viewMode === mode)?.[1]}</strong>
|
||||
<span>{modeExplanation(mode)}</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div className="rellis-review__metrics">
|
||||
<article>
|
||||
<span>Оценимых точек preview</span>
|
||||
<strong>{formatPercent(evaluatedPoints / preview.pointCount)}</strong>
|
||||
<small>ignore не участвует в метриках</small>
|
||||
</article>
|
||||
<article>
|
||||
<span>Ground среди оценимых</span>
|
||||
<strong>
|
||||
{formatPercent(evaluatedPoints ? groundPoints / evaluatedPoints : 0)}
|
||||
</strong>
|
||||
<small>версионная mapping policy v1</small>
|
||||
</article>
|
||||
<article>
|
||||
<span>Классов в этом scan</span>
|
||||
<strong>{preview.classes.length}</strong>
|
||||
<small>из 20 классов ontology</small>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<section className="rellis-review__policy" aria-label="Ground target mapping">
|
||||
<header>
|
||||
<span className="section-eyebrow">GROUND TARGET POLICY V1</span>
|
||||
<h3>Что именно будет оцениваться</h3>
|
||||
<p>
|
||||
Mapping принадлежит Mission Core и версионируется отдельно от
|
||||
ontology датасета. Вода, лужи, void, sky и неопределённый object не
|
||||
засчитываются ни как ground, ни как obstacle.
|
||||
</p>
|
||||
</header>
|
||||
<div>
|
||||
<article>
|
||||
<span>Ground</span>
|
||||
<strong>{groundClasses.map((item) => item.className).join(" · ") || "—"}</strong>
|
||||
</article>
|
||||
<article>
|
||||
<span>Non-ground</span>
|
||||
<strong>
|
||||
{nonGroundClasses.map((item) => item.className).join(" · ") || "—"}
|
||||
</strong>
|
||||
</article>
|
||||
<article>
|
||||
<span>Ignore</span>
|
||||
<strong>{ignoredClasses.map((item) => item.className).join(" · ") || "—"}</strong>
|
||||
</article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<p className="dataset-sequence-review__raw-note">
|
||||
Лицензия RELLIS-3D — CC-BY-NC-SA-3.0: источник годится для независимой
|
||||
исследовательской проверки, но коммерческое использование данных или
|
||||
производных моделей требует отдельного лицензионного решения.
|
||||
</p>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -35,7 +35,7 @@ after(async () => {
|
|||
|
||||
function catalog(overrides = {}) {
|
||||
return {
|
||||
schema_version: "missioncore.dataset-gateway-catalog/v2",
|
||||
schema_version: "missioncore.dataset-gateway-catalog/v3",
|
||||
access: "read-only",
|
||||
storage: {
|
||||
configured: false,
|
||||
|
|
@ -49,19 +49,60 @@ function catalog(overrides = {}) {
|
|||
},
|
||||
sources: [{
|
||||
source_id: "goose-3d/v2025-08-22",
|
||||
source_kind: "goose",
|
||||
display_name: "GOOSE 3D",
|
||||
role: "primary-offroad-semantic-baseline",
|
||||
license: "CC-BY-SA-4.0",
|
||||
commercial_use: "allowed-with-share-alike",
|
||||
format: "semantickitti-xyzi-label",
|
||||
frame_semantics: "one-lidar-revolution",
|
||||
sensor: "VLS-128",
|
||||
platforms: ["MuCAR-3", "ALICE", "Spot"],
|
||||
annotations: ["semantic-point", "instance-point"],
|
||||
superclasses: ["natural-ground", "obstacle"],
|
||||
download: {
|
||||
automatic: false,
|
||||
reason: "operator-admitted-large-artifact-only",
|
||||
smoke_example_mb: null,
|
||||
primary_scan_archive_gb: 3.3,
|
||||
label_archive_gb: null,
|
||||
poses_archive_gb: null,
|
||||
validation_archive_gb: 3.3,
|
||||
},
|
||||
statistics: {
|
||||
fragment_count: 8,
|
||||
annotated_scan_count: 961,
|
||||
},
|
||||
admission: {
|
||||
status: "blocked-storage-policy",
|
||||
archive: null,
|
||||
frame: null,
|
||||
},
|
||||
}, {
|
||||
source_id: "rellis-3d/v1.1",
|
||||
source_kind: "rellis",
|
||||
display_name: "RELLIS-3D",
|
||||
role: "independent-offroad-cross-dataset-check",
|
||||
license: "CC-BY-NC-SA-3.0",
|
||||
commercial_use: "research-only-license-review-required",
|
||||
format: "semantickitti-xyzi-label",
|
||||
frame_semantics: "one-lidar-revolution",
|
||||
sensor: "Ouster OS1 64",
|
||||
platforms: ["Clearpath Warthog"],
|
||||
annotations: ["semantic-point"],
|
||||
superclasses: ["ground", "obstacle", "ignore"],
|
||||
download: {
|
||||
automatic: false,
|
||||
reason: "operator-admitted-large-artifact-only",
|
||||
smoke_example_mb: 24,
|
||||
primary_scan_archive_gb: 14,
|
||||
label_archive_gb: 0.174,
|
||||
poses_archive_gb: 0.174,
|
||||
},
|
||||
statistics: {
|
||||
fragment_count: 5,
|
||||
annotated_scan_count: 13_556,
|
||||
},
|
||||
admission: {
|
||||
status: "blocked-storage-policy",
|
||||
archive: null,
|
||||
|
|
@ -115,7 +156,8 @@ test("parses three distinct LiDAR representations and current-input boundary", (
|
|||
);
|
||||
assert.equal(parsed.representations[2].accumulation, true);
|
||||
assert.equal(parsed.currentInput.admittedForPatchworkpp, false);
|
||||
assert.equal(parsed.source.frameSemantics, "one-lidar-revolution");
|
||||
assert.equal(parsed.sources[0].frameSemantics, "one-lidar-revolution");
|
||||
assert.equal(parsed.sources[1].sourceKind, "rellis");
|
||||
});
|
||||
|
||||
test("rejects path exposure and upgraded K1 semantics", () => {
|
||||
|
|
@ -147,7 +189,7 @@ test("fetches the read-only gateway endpoint", async () => {
|
|||
});
|
||||
assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway");
|
||||
assert.equal(calls[0].init.method, "GET");
|
||||
assert.equal(parsed.source.displayName, "GOOSE 3D");
|
||||
assert.equal(parsed.sources[0].displayName, "GOOSE 3D");
|
||||
});
|
||||
|
||||
test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||
|
|
@ -182,10 +224,19 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
|||
assert.equal(parsed.pointCount, 2);
|
||||
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
|
||||
|
||||
let requestedUrl = null;
|
||||
const fetched = await fetchDatasetNativeScanPreview({
|
||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
||||
fetcher: async (url) => {
|
||||
requestedUrl = url;
|
||||
return new Response(JSON.stringify(payload), { status: 200 });
|
||||
},
|
||||
});
|
||||
assert.equal(fetched.frameId, "frame-1");
|
||||
assert.deepEqual(fetched.evaluationMask, [1, 1]);
|
||||
assert.equal(
|
||||
requestedUrl,
|
||||
"/api/v1/lidar/dataset-gateway/preview?source_id=goose-3d%2Fv2025-08-22",
|
||||
);
|
||||
|
||||
payload.semantic_rgb_0_to_255.pop();
|
||||
assert.throws(
|
||||
|
|
@ -194,6 +245,92 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
|||
);
|
||||
});
|
||||
|
||||
test("decodes the RELLIS compatibility smoke with explicit ignore mask", async () => {
|
||||
const payload = {
|
||||
schema_version: "missioncore.dataset-native-scan-preview/v2",
|
||||
source_id: "rellis-3d/v1.1",
|
||||
frame_id: "000104",
|
||||
representation: "native-scan",
|
||||
sampling: "deterministic-even-index",
|
||||
source_point_count: 3,
|
||||
point_count: 3,
|
||||
points_xyz_m: [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
|
||||
remission_0_to_255: [0, 127, 255],
|
||||
semantic_label_ids: [3, 4, 31],
|
||||
semantic_rgb_0_to_255: [0, 102, 0, 0, 255, 0, 239, 255, 134],
|
||||
ground_truth_ground: [1, 0, 0],
|
||||
evaluation_mask: [1, 1, 0],
|
||||
classes: [
|
||||
{
|
||||
label_id: 3,
|
||||
class_name: "grass",
|
||||
hex: "#006600",
|
||||
ground_target: "ground",
|
||||
source_point_count: 1,
|
||||
},
|
||||
{
|
||||
label_id: 4,
|
||||
class_name: "tree",
|
||||
hex: "#00ff00",
|
||||
ground_target: "non-ground",
|
||||
source_point_count: 1,
|
||||
},
|
||||
{
|
||||
label_id: 31,
|
||||
class_name: "puddle",
|
||||
hex: "#efff86",
|
||||
ground_target: "ignore",
|
||||
source_point_count: 1,
|
||||
},
|
||||
],
|
||||
coordinate_frame: {
|
||||
frame_id: "sensor/lidar/os1",
|
||||
handedness: "right",
|
||||
x: "forward",
|
||||
y: "left",
|
||||
z: "up",
|
||||
transform_applied: false,
|
||||
},
|
||||
ground_policy: {
|
||||
schema_version: "missioncore.rellis-ground-target-policy/v1",
|
||||
ground: ["grass"],
|
||||
non_ground: ["tree"],
|
||||
ignore: ["puddle"],
|
||||
},
|
||||
source_evidence: {
|
||||
repository_url: "https://github.com/unmannedlab/RELLIS-3D",
|
||||
repository_commit: "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
|
||||
label_config_sha256: "5".repeat(64),
|
||||
point_sha256: "e".repeat(64),
|
||||
label_sha256: "9".repeat(64),
|
||||
license: "CC-BY-NC-SA-3.0",
|
||||
},
|
||||
safety: {
|
||||
visualization_only: true,
|
||||
compatibility_smoke_only: true,
|
||||
navigation_or_safety_accepted: false,
|
||||
},
|
||||
};
|
||||
const parsed = parseDatasetNativeScanPreview(payload);
|
||||
assert.equal(parsed.sourceId, "rellis-3d/v1.1");
|
||||
assert.deepEqual(parsed.evaluationMask, [1, 1, 0]);
|
||||
assert.equal(parsed.classes[2].groundTarget, "ignore");
|
||||
assert.equal(parsed.coordinateFrame.frameId, "sensor/lidar/os1");
|
||||
|
||||
let requestedUrl = null;
|
||||
await fetchDatasetNativeScanPreview({
|
||||
sourceId: "rellis-3d/v1.1",
|
||||
fetcher: async (url) => {
|
||||
requestedUrl = url;
|
||||
return new Response(JSON.stringify(payload), { status: 200 });
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
requestedUrl,
|
||||
"/api/v1/lidar/dataset-gateway/preview?source_id=rellis-3d%2Fv1.1",
|
||||
);
|
||||
});
|
||||
|
||||
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
|
||||
const payload = {
|
||||
schema_version: "missioncore.dataset-ground-comparison-preview/v2",
|
||||
|
|
|
|||
|
|
@ -203,6 +203,14 @@ qualification bootstrap, never the product data plane. A shared
|
|||
operator-facing deployment still requires authentication/RBAC, service
|
||||
supervision and an accepted routed worker transport.
|
||||
|
||||
The Dataset Gateway is now multi-source. RELLIS S0 pins official repository
|
||||
frame `000104`, verifies `131,072` Ouster XYZI points against aligned labels and
|
||||
publishes a bounded semantic/ground-target preview beside GOOSE in the same
|
||||
`Полигон → Датасеты` workflow. This closes only reader, axes, ontology,
|
||||
licensing and versioned ignore-policy compatibility. Full RELLIS Ouster scans,
|
||||
labels and poses remain D-only admission work before the cross-dataset
|
||||
Current/Patchwork++ decision.
|
||||
|
||||
Mission Core remains a distributed web product. React is served by a long-lived
|
||||
Gateway/API and renders canonical state through browser-native components.
|
||||
Gazebo/PX4/ROS 2/Nav2 run headless on a registered Linux simulation worker; the
|
||||
|
|
|
|||
|
|
@ -878,6 +878,11 @@ workflow:
|
|||
- `Датасеты → Открыть датасет` shows the admitted source. GOOSE validation is
|
||||
explicitly grouped into eight independent fragments; Play is bounded to one
|
||||
fragment and described as accelerated review of sparse annotated scans.
|
||||
- The catalog is multi-source. RELLIS-3D v1.1 appears as an independent Ouster
|
||||
domain with its own license and admission state. Its current `smoke-ready`
|
||||
view shows official frame `000104`, semantic colors, FLU axes and the
|
||||
versioned `ground / non-ground / ignore` mapping; it does not show invented
|
||||
algorithm metrics.
|
||||
- The dataset viewer exposes source frame number, nanosecond timestamp gaps,
|
||||
ground truth and Current/Patchwork++ overlays.
|
||||
- The fixed sensor-centric coordinate frame and operator camera are preserved
|
||||
|
|
@ -894,6 +899,12 @@ The accepted `3.3 GB` annotated validation ZIP is not a continuous recording.
|
|||
Continuous ego-motion requires a separately admitted raw ROS bag and
|
||||
localization source.
|
||||
|
||||
RELLIS S0 uses only a bounded derivative of the pinned official example for
|
||||
browser inspection. The full `14 GB` Ouster SemanticKITTI scans, labels and
|
||||
poses remain a separate D-only admission gate. Its `CC-BY-NC-SA-3.0` license
|
||||
means cross-dataset research evidence cannot silently become an unrestricted
|
||||
commercial runtime or training dependency.
|
||||
|
||||
The backend reads the configured repository from
|
||||
`MISSIONCORE_POLYGON_RUNS_ROOT` using
|
||||
`QualificationRunStore(read_only=True)`. It exposes only:
|
||||
|
|
|
|||
|
|
@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
|
|||
domain adaptation after a public baseline proves that the pipeline and metric
|
||||
harness work.
|
||||
|
||||
### L2.5 — Dataset Gateway — first-frame A/B complete
|
||||
### L2.5 — Dataset Gateway — GOOSE qualification and RELLIS S0 complete
|
||||
|
||||
- [x] Define separate `native-scan`, `normalized-scan` and
|
||||
`rolling-local-map` representations.
|
||||
|
|
@ -317,12 +317,21 @@ harness work.
|
|||
through an immutable R1 Polygon replay-shadow run.
|
||||
- [x] Add named range/FOV/density/noise/dropout degradation profiles without
|
||||
overwriting the native frame.
|
||||
- [x] Upgrade the gateway to two independently identified sources instead of
|
||||
treating GOOSE as a universal dataset contract.
|
||||
- [x] Pin and verify the official RELLIS-3D Ouster example, preserve its
|
||||
`131,072` point/label alignment and publish a bounded semantic viewer.
|
||||
- [x] Version the RELLIS `ground / non-ground / ignore` mapping and expose
|
||||
licensing as research-only evidence.
|
||||
- [ ] Admit the full RELLIS Ouster SemanticKITTI scans, labels and poses on
|
||||
worker D, then run the same Current/Patchwork++ harness.
|
||||
|
||||
The architectural contract and run sequence are fixed in
|
||||
`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its
|
||||
published 3D format is one LiDAR revolution with point-wise semantic and
|
||||
instance labels in off-road environments. RELLIS-3D remains the second-source
|
||||
cross-check after the GOOSE harness is stable.
|
||||
instance labels in off-road environments. The RELLIS-3D compatibility smoke is
|
||||
now complete against official frame `000104`; the full second-source algorithm
|
||||
cross-check remains the next gate.
|
||||
|
||||
### L3 — LiDAR-native 3D detection
|
||||
|
||||
|
|
@ -394,7 +403,10 @@ The near-term value is not a prettier point cloud:
|
|||
The GOOSE validation-split and deterministic degradation gates are complete:
|
||||
pinned Patchwork++ improves micro Ground IoU from `47.42%` to `66.40%`, raises
|
||||
natural-ground recall from `51.76%` to `76.22%` and stays below `23.41 ms` p95
|
||||
across 961 frames. The decision remains shadow-only. The highest-value
|
||||
immediate work is now a second-source cross-check plus LiDAR-native detection
|
||||
on the same gateway. Nvblox and alternative SLAM remain later because their
|
||||
timing, pose and scan-geometry gates are not yet satisfied.
|
||||
across 961 frames. RELLIS S0 proves that the shared reader, axes, ontology and
|
||||
ground-target mapping work on a different Ouster off-road domain; it does not
|
||||
yet score an algorithm. The decision remains shadow-only. The highest-value
|
||||
immediate work is full RELLIS ground qualification, followed by the real
|
||||
sensor input contract and LiDAR-native detection on the same gateway. Nvblox
|
||||
and alternative SLAM remain later because their timing, pose and scan-geometry
|
||||
gates are not yet satisfied.
|
||||
|
|
|
|||
|
|
@ -19,12 +19,13 @@ datasets and future real onboard sensors.
|
|||
|
||||
Implemented now:
|
||||
|
||||
- `missioncore.dataset-gateway-catalog/v2`, exposed read-only at
|
||||
- multi-source `missioncore.dataset-gateway-catalog/v3`, exposed read-only at
|
||||
`GET /api/v1/lidar/dataset-gateway`;
|
||||
- path-free `missioncore.dataset-admission/v1` worker evidence instead of a
|
||||
static React status;
|
||||
- a bounded `missioncore.dataset-native-scan-preview/v1`, exposed read-only at
|
||||
`GET /api/v1/lidar/dataset-gateway/preview`;
|
||||
- bounded `missioncore.dataset-native-scan-preview/v1` (GOOSE) and `/v2`
|
||||
(RELLIS smoke) artifacts, selected by source at
|
||||
`GET /api/v1/lidar/dataset-gateway/preview?source_id=...`;
|
||||
- explicit `native-scan`, `normalized-scan` and `rolling-local-map`
|
||||
representations;
|
||||
- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI
|
||||
|
|
@ -39,6 +40,10 @@ Implemented now:
|
|||
`Открыть` action;
|
||||
- one admitted GOOSE validation frame with source colors, normalized remission
|
||||
and an independent ground-truth view;
|
||||
- a pinned RELLIS-3D v1.1 compatibility smoke using the official Ouster example
|
||||
frame `000104`, its native `131,072` XYZI points, point-aligned labels,
|
||||
official ontology colors, explicit FLU sensor axes and a versioned
|
||||
`ground / non-ground / ignore` evaluation policy;
|
||||
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
||||
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
||||
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
||||
|
|
@ -54,6 +59,8 @@ Not implemented:
|
|||
- no implicit coordinate conversion;
|
||||
- no fake ring/timestamp reconstruction for K1 MQTT evidence;
|
||||
- no model training or production promotion;
|
||||
- no full RELLIS Ouster archive, pose set, algorithm comparison or ROS bag
|
||||
admission yet;
|
||||
- no rolling-map implementation yet.
|
||||
|
||||
## Product surface boundary
|
||||
|
|
@ -73,10 +80,11 @@ The gateway is not part of device quality diagnostics.
|
|||
backend evidence contract. They are not a second operator navigation item
|
||||
and are not presented as vehicle motion.
|
||||
|
||||
Before real dataset bytes exist, the catalog explains the blocked storage gate
|
||||
and keeps `Открыть датасет` disabled. It becomes available
|
||||
only after the worker manifest says `frame-ready` and the bounded preview passes
|
||||
its own point-alignment and safety checks.
|
||||
Before compatible source evidence exists, the catalog explains the blocked
|
||||
storage gate and keeps `Открыть датасет` disabled. GOOSE becomes available only
|
||||
after the worker manifest says `frame-ready`; RELLIS becomes available at
|
||||
`smoke-ready` only for the pinned official example. Both bounded previews must
|
||||
pass their own identity, point-alignment and safety checks.
|
||||
|
||||
## Why public recordings look different
|
||||
|
||||
|
|
@ -191,8 +199,53 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
|||
- [x] Patchwork++ one-frame accuracy measured against ground truth.
|
||||
- [x] Current/Patchwork++ validation-split gate qualified.
|
||||
- [x] Sensor-degradation matrix qualified.
|
||||
- [x] Dataset catalog accepts more than one independently identified source.
|
||||
- [x] Official RELLIS Ouster example passes the shared SemanticKITTI reader.
|
||||
- [x] RELLIS axes, ontology colors and versioned ground-target/ignore mapping
|
||||
are visible in the same Dataset workflow.
|
||||
- [ ] Full RELLIS Ouster SemanticKITTI scans, labels and poses admitted on
|
||||
worker D.
|
||||
- [ ] Current/Patchwork++ RELLIS comparison qualified without obstacle loss.
|
||||
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
||||
|
||||
## RELLIS S0 compatibility evidence
|
||||
|
||||
RELLIS is the independent second-source check, not extra decoration in the
|
||||
catalog. GOOSE and RELLIS share the ingress format and viewer, but retain
|
||||
separate source identity, sensor domain, ontology, licensing and results.
|
||||
|
||||
The pinned smoke input is the official repository example at commit
|
||||
`c17a118fcaed1559f03cc32cc3a91dedc557f8b8`:
|
||||
|
||||
- point file: `utils/example/000104.bin`;
|
||||
- label file: `utils/example/000104.label`;
|
||||
- source points: `131,072`;
|
||||
- point SHA-256:
|
||||
`ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af`;
|
||||
- label SHA-256:
|
||||
`9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e`;
|
||||
- official ontology config SHA-256:
|
||||
`573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad`;
|
||||
- bounded browser preview: `20,000` deterministic even-index points;
|
||||
- coordinate frame: right-handed, `x` forward, `y` left, `z` up; no vehicle
|
||||
transform, pose registration or deskew is claimed;
|
||||
- license: `CC-BY-NC-SA-3.0`; this is a research qualification source, not an
|
||||
unrestricted commercial runtime dependency.
|
||||
|
||||
Mission Core ground policy v1 treats `dirt`, `grass`, `asphalt`, `concrete` and
|
||||
`mud` as ground; clear structures, vegetation, people, vehicles and rubble as
|
||||
non-ground; and `void`, `water`, `sky`, generic `object` and `puddle` as
|
||||
`ignore`. The ignore set is excluded from future metrics instead of being
|
||||
silently counted as either free ground or obstacle. This mapping is a Mission
|
||||
Core evaluation decision, not an upstream RELLIS claim.
|
||||
|
||||
The smoke closes reader, alignment, axes, palette and mapping compatibility
|
||||
only. The next gate admits the `14 GB` Ouster SemanticKITTI scans plus the
|
||||
`174 MB` labels and `174 MB` poses to D-only worker storage. Full ROS bags are
|
||||
not needed for the ground cross-check. Patchwork++ may be compared only after a
|
||||
physical Ouster mounting-height profile is evidenced; no height is inferred
|
||||
from the example cloud.
|
||||
|
||||
## First GOOSE admission evidence
|
||||
|
||||
- worker root: canonical D-only root, path never exposed by the HTTP API;
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ first admitted source because it publishes off-road point-wise semantic and
|
|||
instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its
|
||||
annotated point-cloud file represents one LiDAR revolution.
|
||||
|
||||
The gateway contract is multi-source, not GOOSE-shaped. RELLIS-3D v1.1 is the
|
||||
second source: it uses the same lossless SemanticKITTI ingress but keeps a
|
||||
separate Ouster OS1 domain, ontology, admission state, ground-target mapping,
|
||||
license and result set. Shared format never implies shared calibration,
|
||||
mounting height or commercial-use rights.
|
||||
|
||||
The gateway:
|
||||
|
||||
- preserves the native GOOSE frame before adaptation;
|
||||
|
|
@ -48,6 +54,10 @@ The gateway:
|
|||
- refuses automatic downloads of large archives;
|
||||
- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror;
|
||||
- never promotes K1 `lio_pcl` to `native-scan`.
|
||||
- excludes explicitly ambiguous RELLIS classes through a versioned `ignore`
|
||||
mask instead of silently scoring them as ground or obstacle;
|
||||
- exposes the RELLIS `CC-BY-NC-SA-3.0` restriction as research evidence and
|
||||
never treats it as an unrestricted production dependency.
|
||||
|
||||
Large-artifact execution remains worker-local. The browser receives only a
|
||||
path-free admission manifest and a bounded visualization preview. The canonical
|
||||
|
|
@ -82,6 +92,8 @@ unavailable rather than inventing timestamps.
|
|||
- Until bytes are installed, the UI exposes an explicit empty state. After a
|
||||
`frame-ready` worker admission, the same surface becomes an interactive
|
||||
source/remission/ground-truth viewer without changing the representation.
|
||||
- A bounded `smoke-ready` source may prove reader, axes, ontology and mapping
|
||||
compatibility without claiming full-dataset or algorithm qualification.
|
||||
- Dataset and device inputs can share downstream algorithms only after their
|
||||
normalized contracts match.
|
||||
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
||||
|
|
@ -115,6 +127,7 @@ unavailable rather than inventing timestamps.
|
|||
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
|
||||
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
|
||||
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
|
||||
- [RELLIS-3D official repository](https://github.com/unmannedlab/RELLIS-3D)
|
||||
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
||||
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
||||
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,15 @@ from k1link.datasets.goose_qualification import (
|
|||
GroundAcceptancePolicy,
|
||||
qualify_goose_ground,
|
||||
)
|
||||
from k1link.datasets.rellis_smoke import (
|
||||
RELLIS_CLASSES,
|
||||
RELLIS_GROUND_POLICY_SCHEMA,
|
||||
RELLIS_PREVIEW_SCHEMA,
|
||||
RELLIS_SOURCE_ID,
|
||||
RellisSmokeError,
|
||||
build_rellis_official_smoke_preview,
|
||||
rellis_native_scan_preview,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DATASET_GATEWAY_CATALOG_SCHEMA",
|
||||
|
|
@ -54,12 +63,18 @@ __all__ = [
|
|||
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
|
||||
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
||||
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
||||
"RELLIS_CLASSES",
|
||||
"RELLIS_GROUND_POLICY_SCHEMA",
|
||||
"RELLIS_PREVIEW_SCHEMA",
|
||||
"RELLIS_SOURCE_ID",
|
||||
"RellisSmokeError",
|
||||
"GoosePatchworkProfile",
|
||||
"GroundAcceptancePolicy",
|
||||
"DegradationProfile",
|
||||
"DEFAULT_DEGRADATIONS",
|
||||
"benchmark_goose_current_ground",
|
||||
"benchmark_goose_patchwork_ground",
|
||||
"build_rellis_official_smoke_preview",
|
||||
"configured_dataset_admission_manifest",
|
||||
"configured_dataset_ground_preview",
|
||||
"configured_dataset_preview",
|
||||
|
|
@ -69,5 +84,6 @@ __all__ = [
|
|||
"read_dataset_ground_preview",
|
||||
"read_dataset_native_scan_preview",
|
||||
"read_semantic_kitti_frame",
|
||||
"rellis_native_scan_preview",
|
||||
"qualify_goose_ground",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
|
|||
)
|
||||
from k1link.datasets.goose_qualification import qualify_goose_ground
|
||||
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
||||
from k1link.datasets.rellis_smoke import (
|
||||
RellisSmokeError,
|
||||
build_rellis_official_smoke_preview,
|
||||
)
|
||||
|
||||
app = typer.Typer(
|
||||
add_completion=False,
|
||||
|
|
@ -158,5 +162,50 @@ def build_goose_ground_review_command(
|
|||
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
@app.command("build-rellis-smoke-preview")
|
||||
def build_rellis_smoke_preview_command(
|
||||
points: Annotated[
|
||||
Path,
|
||||
typer.Option("--points", exists=True, dir_okay=False, resolve_path=True),
|
||||
],
|
||||
labels: Annotated[
|
||||
Path,
|
||||
typer.Option("--labels", exists=True, dir_okay=False, resolve_path=True),
|
||||
],
|
||||
out: Annotated[
|
||||
Path,
|
||||
typer.Option("--out", dir_okay=False, resolve_path=True),
|
||||
],
|
||||
preview_points: Annotated[
|
||||
int,
|
||||
typer.Option("--preview-points", min=1, max=50_000),
|
||||
] = 20_000,
|
||||
) -> None:
|
||||
"""Verify the pinned official RELLIS example and publish a bounded preview."""
|
||||
|
||||
try:
|
||||
preview = build_rellis_official_smoke_preview(
|
||||
points,
|
||||
labels,
|
||||
out,
|
||||
preview_points=preview_points,
|
||||
)
|
||||
except RellisSmokeError as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(
|
||||
json.dumps(
|
||||
{
|
||||
"source_id": preview["source_id"],
|
||||
"frame_id": preview["frame_id"],
|
||||
"source_point_count": preview["source_point_count"],
|
||||
"preview_point_count": preview["point_count"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import numpy.typing as npt
|
|||
|
||||
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
|
||||
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v3"
|
||||
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||
|
|
@ -177,6 +177,7 @@ def _is_worker_d_storage(root: Path | None) -> bool:
|
|||
def dataset_gateway_catalog(
|
||||
dataset_root: Path | None = None,
|
||||
admission_manifest_path: Path | None = None,
|
||||
rellis_preview_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||
|
||||
|
|
@ -196,6 +197,14 @@ def dataset_gateway_catalog(
|
|||
locally_admitted = _is_worker_d_storage(root)
|
||||
worker_admitted = bool(admission and admission["storage"]["admitted"])
|
||||
storage_admitted = locally_admitted or worker_admitted
|
||||
rellis_preview: dict[str, Any] | None = None
|
||||
if rellis_preview_path is not None and rellis_preview_path.is_file():
|
||||
try:
|
||||
candidate = read_dataset_native_scan_preview(rellis_preview_path)
|
||||
if candidate.get("source_id") == "rellis-3d/v1.1":
|
||||
rellis_preview = candidate
|
||||
except DatasetAdmissionError:
|
||||
pass
|
||||
source_status = (
|
||||
str(admission["status"])
|
||||
if admission is not None
|
||||
|
|
@ -212,6 +221,20 @@ def dataset_gateway_catalog(
|
|||
if storage_admitted
|
||||
else "configure-dataset-root-on-worker-d"
|
||||
)
|
||||
rellis_status = (
|
||||
"smoke-ready"
|
||||
if rellis_preview is not None
|
||||
else "ready-for-smoke"
|
||||
if storage_admitted
|
||||
else "blocked-storage-policy"
|
||||
)
|
||||
catalog_next_action = (
|
||||
"admit-rellis-ouster-semantickitti-to-worker-d"
|
||||
if rellis_preview is not None and source_status == "frame-ready"
|
||||
else "build-rellis-official-example-smoke"
|
||||
if source_status == "frame-ready"
|
||||
else next_action
|
||||
)
|
||||
return {
|
||||
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||
"access": "read-only",
|
||||
|
|
@ -234,11 +257,14 @@ def dataset_gateway_catalog(
|
|||
"sources": [
|
||||
{
|
||||
"source_id": "goose-3d/v2025-08-22",
|
||||
"source_kind": "goose",
|
||||
"display_name": "GOOSE 3D",
|
||||
"role": "primary-offroad-semantic-baseline",
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"commercial_use": "allowed-with-share-alike",
|
||||
"format": "semantickitti-xyzi-label",
|
||||
"frame_semantics": "one-lidar-revolution",
|
||||
"sensor": "VLS-128",
|
||||
"platforms": ["MuCAR-3", "ALICE", "Spot"],
|
||||
"annotations": ["semantic-point", "instance-point"],
|
||||
"superclasses": [
|
||||
|
|
@ -255,10 +281,18 @@ def dataset_gateway_catalog(
|
|||
"download": {
|
||||
"automatic": False,
|
||||
"reason": "operator-admitted-large-artifact-only",
|
||||
"smoke_example_mb": None,
|
||||
"primary_scan_archive_gb": 3.3,
|
||||
"label_archive_gb": None,
|
||||
"poses_archive_gb": None,
|
||||
"training_archive_gb": 27.0,
|
||||
"validation_archive_gb": 3.3,
|
||||
"test_archive_gb": 3.3,
|
||||
},
|
||||
"statistics": {
|
||||
"fragment_count": 8,
|
||||
"annotated_scan_count": 961,
|
||||
},
|
||||
"admission": {
|
||||
"status": source_status,
|
||||
"native_scan": "ready-after-download",
|
||||
|
|
@ -267,7 +301,72 @@ def dataset_gateway_catalog(
|
|||
"archive": admission["archive"] if admission is not None else None,
|
||||
"frame": admission["frame"] if admission is not None else None,
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"source_id": "rellis-3d/v1.1",
|
||||
"source_kind": "rellis",
|
||||
"display_name": "RELLIS-3D",
|
||||
"role": "independent-offroad-cross-dataset-check",
|
||||
"license": "CC-BY-NC-SA-3.0",
|
||||
"commercial_use": "research-only-license-review-required",
|
||||
"format": "semantickitti-xyzi-label",
|
||||
"frame_semantics": "one-lidar-revolution",
|
||||
"sensor": "Ouster OS1 64",
|
||||
"platforms": ["Clearpath Warthog"],
|
||||
"annotations": ["semantic-point"],
|
||||
"superclasses": [
|
||||
"ground",
|
||||
"vegetation",
|
||||
"structure",
|
||||
"obstacle",
|
||||
"vehicle",
|
||||
"human",
|
||||
"water",
|
||||
"ignore",
|
||||
],
|
||||
"download": {
|
||||
"automatic": False,
|
||||
"reason": "operator-admitted-large-artifact-only",
|
||||
"smoke_example_mb": 24.0,
|
||||
"primary_scan_archive_gb": 14.0,
|
||||
"label_archive_gb": 0.174,
|
||||
"poses_archive_gb": 0.174,
|
||||
"training_archive_gb": None,
|
||||
"validation_archive_gb": None,
|
||||
"test_archive_gb": None,
|
||||
},
|
||||
"statistics": {
|
||||
"fragment_count": 5,
|
||||
"annotated_scan_count": 13_556,
|
||||
},
|
||||
"admission": {
|
||||
"status": rellis_status,
|
||||
"native_scan": (
|
||||
"official-example-compatible"
|
||||
if rellis_preview is not None
|
||||
else "requires-official-example-smoke"
|
||||
),
|
||||
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
|
||||
"rolling_local_map": "requires-poses-timing-and-map-policy",
|
||||
"archive": None,
|
||||
"frame": (
|
||||
{
|
||||
"frame_id": rellis_preview["frame_id"],
|
||||
"point_count": rellis_preview["source_point_count"],
|
||||
"semantic_class_count": len(rellis_preview["classes"]),
|
||||
"ground_truth_ground_fraction": (
|
||||
sum(rellis_preview["ground_truth_ground"])
|
||||
/ rellis_preview["point_count"]
|
||||
),
|
||||
"preview_point_count": rellis_preview["point_count"],
|
||||
"preview_sha256": None,
|
||||
"preview_available": True,
|
||||
}
|
||||
if rellis_preview is not None
|
||||
else None
|
||||
),
|
||||
},
|
||||
},
|
||||
],
|
||||
"representations": [
|
||||
{
|
||||
|
|
@ -330,7 +429,7 @@ def dataset_gateway_catalog(
|
|||
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
|
||||
}
|
||||
],
|
||||
"next_action": next_action,
|
||||
"next_action": catalog_next_action,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -417,9 +516,13 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
|||
|
||||
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
|
||||
identity = (document.get("schema_version"), document.get("source_id"))
|
||||
if (
|
||||
document.get("schema_version") != DATASET_PREVIEW_SCHEMA
|
||||
or document.get("source_id") != "goose-3d/v2025-08-22"
|
||||
identity
|
||||
not in {
|
||||
(DATASET_PREVIEW_SCHEMA, "goose-3d/v2025-08-22"),
|
||||
("missioncore.dataset-native-scan-preview/v2", "rellis-3d/v1.1"),
|
||||
}
|
||||
or document.get("representation") != "native-scan"
|
||||
or document.get("sampling") != "deterministic-even-index"
|
||||
):
|
||||
|
|
@ -433,6 +536,11 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
|||
semantic_ids = document.get("semantic_label_ids")
|
||||
semantic_rgb = document.get("semantic_rgb_0_to_255")
|
||||
ground = document.get("ground_truth_ground")
|
||||
evaluated = (
|
||||
document.get("evaluation_mask")
|
||||
if identity[1] == "rellis-3d/v1.1"
|
||||
else [1] * point_count
|
||||
)
|
||||
if (
|
||||
not isinstance(points, list)
|
||||
or len(points) != point_count
|
||||
|
|
@ -444,6 +552,8 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
|||
or len(semantic_rgb) != point_count * 3
|
||||
or not isinstance(ground, list)
|
||||
or len(ground) != point_count
|
||||
or not isinstance(evaluated, list)
|
||||
or len(evaluated) != point_count
|
||||
):
|
||||
raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
|
||||
for point in points:
|
||||
|
|
@ -463,6 +573,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
|||
(semantic_ids, 65_535, "semantic label"),
|
||||
(semantic_rgb, 255, "semantic color"),
|
||||
(ground, 1, "ground mask"),
|
||||
(evaluated, 1, "evaluation mask"),
|
||||
):
|
||||
if any(
|
||||
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
|
||||
|
|
@ -473,16 +584,81 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
|||
if not isinstance(classes, list) or len(classes) > 64:
|
||||
raise DatasetAdmissionError("dataset preview class catalog is incompatible")
|
||||
safety = _object(document.get("safety"), "safety")
|
||||
if safety != {
|
||||
expected_safety = {
|
||||
"visualization_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}:
|
||||
}
|
||||
if identity[1] == "rellis-3d/v1.1":
|
||||
expected_safety["compatibility_smoke_only"] = True
|
||||
if safety != expected_safety:
|
||||
raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
|
||||
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
|
||||
raise DatasetAdmissionError("dataset preview frame id is incompatible")
|
||||
if identity[1] == "rellis-3d/v1.1":
|
||||
_validate_rellis_preview_metadata(document, classes)
|
||||
return document
|
||||
|
||||
|
||||
def _validate_rellis_preview_metadata(
|
||||
document: dict[str, Any],
|
||||
classes: list[Any],
|
||||
) -> None:
|
||||
coordinate_frame = _object(document.get("coordinate_frame"), "coordinate_frame")
|
||||
if coordinate_frame != {
|
||||
"frame_id": "sensor/lidar/os1",
|
||||
"handedness": "right",
|
||||
"x": "forward",
|
||||
"y": "left",
|
||||
"z": "up",
|
||||
"transform_applied": False,
|
||||
}:
|
||||
raise DatasetAdmissionError("RELLIS coordinate frame is incompatible")
|
||||
policy = _object(document.get("ground_policy"), "ground_policy")
|
||||
if (
|
||||
policy.get("schema_version") != "missioncore.rellis-ground-target-policy/v1"
|
||||
or set(policy) != {"schema_version", "ground", "non_ground", "ignore"}
|
||||
or any(
|
||||
not isinstance(policy.get(key), list)
|
||||
or any(not isinstance(value, str) or not value for value in policy[key])
|
||||
for key in ("ground", "non_ground", "ignore")
|
||||
)
|
||||
):
|
||||
raise DatasetAdmissionError("RELLIS ground target policy is incompatible")
|
||||
for item in classes:
|
||||
value = _object(item, "class")
|
||||
if (
|
||||
value.get("ground_target") not in {"ground", "non-ground", "ignore"}
|
||||
or not isinstance(value.get("label_id"), int)
|
||||
or not isinstance(value.get("class_name"), str)
|
||||
or not isinstance(value.get("hex"), str)
|
||||
or not isinstance(value.get("source_point_count"), int)
|
||||
):
|
||||
raise DatasetAdmissionError("RELLIS class catalog is incompatible")
|
||||
evidence = _object(document.get("source_evidence"), "source_evidence")
|
||||
if (
|
||||
set(evidence)
|
||||
!= {
|
||||
"repository_url",
|
||||
"repository_commit",
|
||||
"label_config_sha256",
|
||||
"point_sha256",
|
||||
"label_sha256",
|
||||
"license",
|
||||
}
|
||||
or evidence.get("repository_url") != "https://github.com/unmannedlab/RELLIS-3D"
|
||||
or evidence.get("repository_commit")
|
||||
!= "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
|
||||
or evidence.get("label_config_sha256")
|
||||
!= "573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||
or evidence.get("point_sha256")
|
||||
!= "ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||
or evidence.get("label_sha256")
|
||||
!= "9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||
or evidence.get("license") != "CC-BY-NC-SA-3.0"
|
||||
):
|
||||
raise DatasetAdmissionError("RELLIS source evidence is incompatible")
|
||||
|
||||
|
||||
def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,275 @@
|
|||
"""Pinned RELLIS-3D compatibility smoke for the shared Dataset Gateway.
|
||||
|
||||
The smoke check deliberately stops before algorithm qualification. It proves
|
||||
that Mission Core can read one official Ouster OS1 SemanticKITTI frame, preserve
|
||||
point/label alignment, apply an explicit ground-evaluation policy and publish a
|
||||
bounded visualization artifact. Full RELLIS archives and ROS bags remain
|
||||
worker-D-only inputs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.datasets.gateway import DatasetFrameError, DatasetPointFrame, read_semantic_kitti_frame
|
||||
|
||||
RELLIS_SOURCE_ID: Final = "rellis-3d/v1.1"
|
||||
RELLIS_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v2"
|
||||
RELLIS_GROUND_POLICY_SCHEMA: Final = "missioncore.rellis-ground-target-policy/v1"
|
||||
RELLIS_REPOSITORY_URL: Final = "https://github.com/unmannedlab/RELLIS-3D"
|
||||
RELLIS_REPOSITORY_COMMIT: Final = "c17a118fcaed1559f03cc32cc3a91dedc557f8b8"
|
||||
RELLIS_LABEL_CONFIG_SHA256: Final = (
|
||||
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||
)
|
||||
RELLIS_EXAMPLE_POINTS_SHA256: Final = (
|
||||
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||
)
|
||||
RELLIS_EXAMPLE_LABELS_SHA256: Final = (
|
||||
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||
)
|
||||
RELLIS_EXAMPLE_FRAME_ID: Final = "000104"
|
||||
RELLIS_LICENSE: Final = "CC-BY-NC-SA-3.0"
|
||||
MAX_RELLIS_PREVIEW_POINTS: Final = 50_000
|
||||
|
||||
GroundTarget = Literal["ground", "non-ground", "ignore"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RellisClass:
|
||||
label_id: int
|
||||
class_name: str
|
||||
bgr: tuple[int, int, int]
|
||||
ground_target: GroundTarget
|
||||
|
||||
@property
|
||||
def rgb(self) -> tuple[int, int, int]:
|
||||
blue, green, red = self.bgr
|
||||
return red, green, blue
|
||||
|
||||
@property
|
||||
def hex(self) -> str:
|
||||
red, green, blue = self.rgb
|
||||
return f"#{red:02x}{green:02x}{blue:02x}"
|
||||
|
||||
def to_preview_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"label_id": self.label_id,
|
||||
"class_name": self.class_name,
|
||||
"hex": self.hex,
|
||||
"ground_target": self.ground_target,
|
||||
}
|
||||
|
||||
|
||||
# IDs, names and BGR colors are pinned to the official repository config at
|
||||
# RELLIS_REPOSITORY_COMMIT. The final column is Mission Core's versioned,
|
||||
# algorithm-scoped ground evaluation policy; it is not an upstream claim.
|
||||
RELLIS_CLASSES: Final[tuple[RellisClass, ...]] = (
|
||||
RellisClass(0, "void", (0, 0, 0), "ignore"),
|
||||
RellisClass(1, "dirt", (108, 64, 20), "ground"),
|
||||
RellisClass(3, "grass", (0, 102, 0), "ground"),
|
||||
RellisClass(4, "tree", (0, 255, 0), "non-ground"),
|
||||
RellisClass(5, "pole", (0, 153, 153), "non-ground"),
|
||||
RellisClass(6, "water", (0, 128, 255), "ignore"),
|
||||
RellisClass(7, "sky", (0, 0, 255), "ignore"),
|
||||
RellisClass(8, "vehicle", (255, 255, 0), "non-ground"),
|
||||
RellisClass(9, "object", (255, 0, 127), "ignore"),
|
||||
RellisClass(10, "asphalt", (64, 64, 64), "ground"),
|
||||
RellisClass(12, "building", (255, 0, 0), "non-ground"),
|
||||
RellisClass(15, "log", (102, 0, 0), "non-ground"),
|
||||
RellisClass(17, "person", (204, 153, 255), "non-ground"),
|
||||
RellisClass(18, "fence", (102, 0, 204), "non-ground"),
|
||||
RellisClass(19, "bush", (255, 153, 204), "non-ground"),
|
||||
RellisClass(23, "concrete", (170, 170, 170), "ground"),
|
||||
RellisClass(27, "barrier", (41, 121, 255), "non-ground"),
|
||||
RellisClass(31, "puddle", (134, 255, 239), "ignore"),
|
||||
RellisClass(33, "mud", (99, 66, 34), "ground"),
|
||||
RellisClass(34, "rubble", (110, 22, 138), "non-ground"),
|
||||
)
|
||||
RELLIS_CLASS_BY_ID: Final = {item.label_id: item for item in RELLIS_CLASSES}
|
||||
|
||||
|
||||
class RellisSmokeError(RuntimeError):
|
||||
"""The pinned official example cannot satisfy the RELLIS smoke contract."""
|
||||
|
||||
|
||||
def build_rellis_official_smoke_preview(
|
||||
point_path: Path,
|
||||
label_path: Path,
|
||||
output_path: Path,
|
||||
*,
|
||||
preview_points: int = 20_000,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify the official example and publish one bounded path-free preview."""
|
||||
|
||||
points_digest = _sha256_file(point_path)
|
||||
labels_digest = _sha256_file(label_path)
|
||||
if points_digest != RELLIS_EXAMPLE_POINTS_SHA256:
|
||||
raise RellisSmokeError("RELLIS official example point digest is incompatible")
|
||||
if labels_digest != RELLIS_EXAMPLE_LABELS_SHA256:
|
||||
raise RellisSmokeError("RELLIS official example label digest is incompatible")
|
||||
try:
|
||||
frame = read_semantic_kitti_frame(point_path, label_path)
|
||||
except DatasetFrameError as exc:
|
||||
raise RellisSmokeError("RELLIS official example violates XYZI/label alignment") from exc
|
||||
preview = rellis_native_scan_preview(
|
||||
frame,
|
||||
frame_id=RELLIS_EXAMPLE_FRAME_ID,
|
||||
maximum_points=preview_points,
|
||||
source_evidence={
|
||||
"repository_url": RELLIS_REPOSITORY_URL,
|
||||
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
|
||||
"point_sha256": points_digest,
|
||||
"label_sha256": labels_digest,
|
||||
"license": RELLIS_LICENSE,
|
||||
},
|
||||
)
|
||||
_atomic_json(output_path, preview)
|
||||
return preview
|
||||
|
||||
|
||||
def rellis_native_scan_preview(
|
||||
frame: DatasetPointFrame,
|
||||
*,
|
||||
frame_id: str,
|
||||
maximum_points: int,
|
||||
source_evidence: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
"""Build the common viewer artifact from a validated RELLIS native frame."""
|
||||
|
||||
if not frame_id or not 1 <= maximum_points <= MAX_RELLIS_PREVIEW_POINTS:
|
||||
raise RellisSmokeError("RELLIS preview bounds are incompatible")
|
||||
unknown = sorted(
|
||||
int(value)
|
||||
for value in np.unique(frame.semantic_labels)
|
||||
if int(value) not in RELLIS_CLASS_BY_ID
|
||||
)
|
||||
if unknown:
|
||||
raise RellisSmokeError("RELLIS frame contains labels absent from the pinned ontology")
|
||||
required_evidence = {
|
||||
"repository_url",
|
||||
"repository_commit",
|
||||
"label_config_sha256",
|
||||
"point_sha256",
|
||||
"label_sha256",
|
||||
"license",
|
||||
}
|
||||
if set(source_evidence) != required_evidence or any(
|
||||
not isinstance(value, str) or not value for value in source_evidence.values()
|
||||
):
|
||||
raise RellisSmokeError("RELLIS source evidence is incomplete")
|
||||
|
||||
sample_count = min(frame.point_count, maximum_points)
|
||||
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
|
||||
points = frame.points_xyz_m[indices]
|
||||
remission = frame.remission[indices]
|
||||
semantic = frame.semantic_labels[indices]
|
||||
remission_min = float(np.min(remission))
|
||||
remission_max = float(np.max(remission))
|
||||
remission_span = remission_max - remission_min
|
||||
if remission_span <= 0:
|
||||
remission_u8 = np.zeros(sample_count, dtype=np.uint8)
|
||||
else:
|
||||
remission_u8 = np.rint(
|
||||
(remission - remission_min) / remission_span * 255.0
|
||||
).astype(np.uint8)
|
||||
|
||||
ground = np.zeros(sample_count, dtype=np.uint8)
|
||||
evaluated = np.zeros(sample_count, dtype=np.uint8)
|
||||
colors = np.zeros((sample_count, 3), dtype=np.uint8)
|
||||
for label_id in np.unique(semantic):
|
||||
item = RELLIS_CLASS_BY_ID[int(label_id)]
|
||||
mask = semantic == label_id
|
||||
colors[mask] = item.rgb
|
||||
if item.ground_target != "ignore":
|
||||
evaluated[mask] = 1
|
||||
if item.ground_target == "ground":
|
||||
ground[mask] = 1
|
||||
|
||||
present = Counter(int(value) for value in frame.semantic_labels)
|
||||
present_classes = [
|
||||
{
|
||||
**RELLIS_CLASS_BY_ID[label_id].to_preview_dict(),
|
||||
"source_point_count": count,
|
||||
}
|
||||
for label_id, count in sorted(present.items())
|
||||
]
|
||||
return {
|
||||
"schema_version": RELLIS_PREVIEW_SCHEMA,
|
||||
"source_id": RELLIS_SOURCE_ID,
|
||||
"frame_id": frame_id,
|
||||
"representation": "native-scan",
|
||||
"sampling": "deterministic-even-index",
|
||||
"source_point_count": frame.point_count,
|
||||
"point_count": sample_count,
|
||||
"points_xyz_m": points.tolist(),
|
||||
"remission_0_to_255": remission_u8.tolist(),
|
||||
"semantic_label_ids": semantic.astype(np.uint16).tolist(),
|
||||
"semantic_rgb_0_to_255": colors.reshape(-1).tolist(),
|
||||
"ground_truth_ground": ground.tolist(),
|
||||
"evaluation_mask": evaluated.tolist(),
|
||||
"classes": present_classes,
|
||||
"coordinate_frame": {
|
||||
"frame_id": "sensor/lidar/os1",
|
||||
"handedness": "right",
|
||||
"x": "forward",
|
||||
"y": "left",
|
||||
"z": "up",
|
||||
"transform_applied": False,
|
||||
},
|
||||
"ground_policy": {
|
||||
"schema_version": RELLIS_GROUND_POLICY_SCHEMA,
|
||||
"ground": [
|
||||
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ground"
|
||||
],
|
||||
"non_ground": [
|
||||
item.class_name
|
||||
for item in RELLIS_CLASSES
|
||||
if item.ground_target == "non-ground"
|
||||
],
|
||||
"ignore": [
|
||||
item.class_name for item in RELLIS_CLASSES if item.ground_target == "ignore"
|
||||
],
|
||||
},
|
||||
"source_evidence": source_evidence,
|
||||
"safety": {
|
||||
"visualization_only": True,
|
||||
"compatibility_smoke_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024**2), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
raise RellisSmokeError("RELLIS official example is unavailable") from exc
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
dir=path.parent,
|
||||
encoding="utf-8",
|
||||
delete=False,
|
||||
) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
json.dump(document, temporary, ensure_ascii=False, separators=(",", ":"))
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
|
|
@ -426,6 +426,9 @@ app.include_router(
|
|||
dataset_preview_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json"
|
||||
),
|
||||
dataset_rellis_preview_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
||||
),
|
||||
dataset_ground_preview_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
||||
),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import os
|
|||
import re
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
from typing import Annotated, Any, Final
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
|
||||
|
|
@ -21,6 +21,7 @@ from k1link.compute import (
|
|||
lidar_pack_detail,
|
||||
)
|
||||
from k1link.datasets import (
|
||||
RELLIS_SOURCE_ID,
|
||||
DatasetAdmissionError,
|
||||
configured_dataset_admission_manifest,
|
||||
configured_dataset_ground_preview,
|
||||
|
|
@ -62,6 +63,7 @@ def build_lidar_router(
|
|||
field_review_root_provider: RootProvider = configured_lidar_field_review_root,
|
||||
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
||||
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
||||
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
|
||||
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
||||
) -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||
|
|
@ -70,18 +72,32 @@ def build_lidar_router(
|
|||
def get_dataset_gateway() -> dict[str, object]:
|
||||
return dataset_gateway_catalog(
|
||||
admission_manifest_path=dataset_admission_provider(),
|
||||
rellis_preview_path=dataset_rellis_preview_provider(),
|
||||
)
|
||||
|
||||
@router.get("/dataset-gateway/preview")
|
||||
def get_dataset_gateway_preview() -> dict[str, Any]:
|
||||
path = dataset_preview_provider()
|
||||
def get_dataset_gateway_preview(
|
||||
source_id: Annotated[
|
||||
str,
|
||||
Query(min_length=1, max_length=160),
|
||||
] = "goose-3d/v2025-08-22",
|
||||
) -> dict[str, Any]:
|
||||
if source_id == "goose-3d/v2025-08-22":
|
||||
path = dataset_preview_provider()
|
||||
elif source_id == RELLIS_SOURCE_ID:
|
||||
path = dataset_rellis_preview_provider()
|
||||
else:
|
||||
raise HTTPException(status_code=404, detail="Dataset source не найден.")
|
||||
if path is None or not path.is_file():
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Первый размеченный native scan ещё не импортирован.",
|
||||
detail="Размеченный native scan источника ещё не импортирован.",
|
||||
)
|
||||
try:
|
||||
return read_dataset_native_scan_preview(path)
|
||||
preview = read_dataset_native_scan_preview(path)
|
||||
if preview["source_id"] != source_id:
|
||||
raise DatasetAdmissionError("dataset preview source mismatch")
|
||||
return preview
|
||||
except DatasetAdmissionError as exc:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from fastapi.routing import APIRoute
|
|||
from k1link.datasets import (
|
||||
DatasetAdmissionError,
|
||||
DatasetFrameError,
|
||||
RELLIS_SOURCE_ID,
|
||||
dataset_gateway_catalog,
|
||||
goose_admission,
|
||||
goose_benchmark,
|
||||
|
|
@ -19,6 +20,7 @@ from k1link.datasets import (
|
|||
read_dataset_ground_preview,
|
||||
read_dataset_native_scan_preview,
|
||||
read_semantic_kitti_frame,
|
||||
rellis_native_scan_preview,
|
||||
)
|
||||
from k1link.datasets.goose_admission import admit_goose_validation
|
||||
from k1link.datasets.goose_benchmark import (
|
||||
|
|
@ -111,10 +113,14 @@ def test_dataset_gateway_api_is_read_only_and_path_free(
|
|||
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
|
||||
response = route() # type: ignore[operator]
|
||||
|
||||
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2"
|
||||
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v3"
|
||||
assert response["access"] == "read-only"
|
||||
assert response["storage"]["admitted"] is True
|
||||
assert response["storage"]["path_exposed"] is False
|
||||
assert [source["source_id"] for source in response["sources"]] == [
|
||||
"goose-3d/v2025-08-22",
|
||||
RELLIS_SOURCE_ID,
|
||||
]
|
||||
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
|
||||
response["storage"]["required_wsl_root"], # type: ignore[index]
|
||||
"",
|
||||
|
|
@ -286,6 +292,70 @@ def test_dataset_preview_api_is_read_only_and_integrity_checked(
|
|||
assert response["frame_id"] == "frame-1"
|
||||
|
||||
|
||||
def test_rellis_smoke_preview_keeps_semantickitti_alignment_and_ignore_policy(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
points_path = tmp_path / "000104.bin"
|
||||
labels_path = tmp_path / "000104.label"
|
||||
np.asarray(
|
||||
[
|
||||
[1.0, 0.0, -1.0, 0.1],
|
||||
[2.0, 0.2, -0.8, 0.2],
|
||||
[3.0, -0.2, 0.5, 0.3],
|
||||
[4.0, 0.0, -0.5, 0.4],
|
||||
],
|
||||
dtype="<f4",
|
||||
).tofile(points_path)
|
||||
np.asarray([3, 23, 4, 31], dtype="<u4").tofile(labels_path)
|
||||
frame = read_semantic_kitti_frame(points_path, labels_path)
|
||||
|
||||
preview = rellis_native_scan_preview(
|
||||
frame,
|
||||
frame_id="000104",
|
||||
maximum_points=4,
|
||||
source_evidence={
|
||||
"repository_url": "https://github.com/unmannedlab/RELLIS-3D",
|
||||
"repository_commit": "c17a118fcaed1559f03cc32cc3a91dedc557f8b8",
|
||||
"label_config_sha256": (
|
||||
"573379a232ac561805987466c391a61fd5ad338be7fb9c4c2842f3f28067e0ad"
|
||||
),
|
||||
"point_sha256": (
|
||||
"ed81a9c3636d55b17d78058c72545d5d22419beecf174d50596d23ae178752af"
|
||||
),
|
||||
"label_sha256": (
|
||||
"9b8c65b710873e931af4ac6dfc7d3dd2298696514ab721e50300bd55ad5b634e"
|
||||
),
|
||||
"license": "CC-BY-NC-SA-3.0",
|
||||
},
|
||||
)
|
||||
preview_path = tmp_path / "rellis-preview.json"
|
||||
preview_path.write_text(json.dumps(preview), encoding="utf-8")
|
||||
|
||||
decoded = read_dataset_native_scan_preview(preview_path)
|
||||
|
||||
assert decoded["source_id"] == RELLIS_SOURCE_ID
|
||||
assert decoded["ground_truth_ground"] == [1, 1, 0, 0]
|
||||
assert decoded["evaluation_mask"] == [1, 1, 1, 0]
|
||||
assert decoded["coordinate_frame"]["x"] == "forward"
|
||||
assert decoded["safety"]["compatibility_smoke_only"] is True
|
||||
|
||||
router = build_lidar_router(
|
||||
dataset_admission_provider=lambda: None,
|
||||
dataset_preview_provider=lambda: None,
|
||||
dataset_rellis_preview_provider=lambda: preview_path,
|
||||
)
|
||||
route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview")
|
||||
response = route(source_id=RELLIS_SOURCE_ID) # type: ignore[operator]
|
||||
assert response["frame_id"] == "000104"
|
||||
|
||||
catalog = dataset_gateway_catalog(
|
||||
Path("/mnt/d/NDC_MISSIONCORE/datasets"),
|
||||
rellis_preview_path=preview_path,
|
||||
)
|
||||
rellis = catalog["sources"][1] # type: ignore[index]
|
||||
assert rellis["admission"]["status"] == "smoke-ready" # type: ignore[index]
|
||||
|
||||
|
||||
def test_goose_current_ground_is_scored_against_independent_labels(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
|
|||
Loading…
Reference in New Issue