diff --git a/README.md b/README.md index 5b8c2be..7ae60cd 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,19 @@ performance, direct worker transport and bounded live fan-out remain open. See [ADR 0014](docs/adr/0014-bounded-external-perception-worker.md) and the [external worker contract](docs/10_EXTERNAL_PERCEPTION_WORKER.md). +## LiDAR Dataset Gateway + +Mission Core now distinguishes a single native LiDAR scan, a normalized +sensor-frame scan and a pose-registered rolling local map. The first Dataset +Gateway adapter reads GOOSE/SemanticKITTI XYZI plus point-wise semantic and +instance labels without silently accumulating or transforming them. Current +K1 `lio_pcl` remains a post-LIO vendor-map increment and is not promoted to a +raw scan. Large dataset bytes are admitted only on the Windows/WSL worker under +`D:\NDC_MISSIONCORE\datasets`; no archive is downloaded automatically. + +See [the Dataset Gateway plan](docs/14_LIDAR_DATASET_GATEWAY.md) and +[ADR 0021](docs/adr/0021-dataset-gateway-representation-boundary.md). + ## Simulation Polygon Mission Core now has a parallel Polygon product branch for reproducible diff --git a/apps/control-station/src/core/lidar/datasetGateway.ts b/apps/control-station/src/core/lidar/datasetGateway.ts new file mode 100644 index 0000000..40eef50 --- /dev/null +++ b/apps/control-station/src/core/lidar/datasetGateway.ts @@ -0,0 +1,243 @@ +export type DatasetRepresentationId = + | "native-scan" + | "normalized-scan" + | "rolling-local-map"; + +export interface DatasetGatewayCatalog { + storage: { + configured: boolean; + admitted: boolean; + status: "ready" | "blocked-storage-policy"; + requiredWindowsRoot: string; + requiredWslRoot: string; + }; + source: { + sourceId: string; + displayName: string; + role: string; + license: string; + format: string; + frameSemantics: "one-lidar-revolution"; + platforms: string[]; + superclasses: string[]; + validationArchiveGb: number; + admissionStatus: "ready-for-download" | "blocked-storage-policy"; + }; + representations: Array<{ + id: DatasetRepresentationId; + title: string; + purpose: string; + accumulation: boolean; + }>; + pipeline: Array<{ + stage: string; + requires: string[]; + produces: string; + }>; + currentInput: { + representation: "vendor-mapped-increment"; + nativeScan: false; + perPointTime: false; + ringOrLine: false; + admittedForPatchworkpp: false; + reason: string; + }; + nextAction: string; +} + +export class DatasetGatewayContractError extends Error {} + +type DatasetFetch = ( + input: RequestInfo | URL, + init?: RequestInit, +) => Promise; + +const SAFE_ID = /^[a-z0-9][a-z0-9._:/-]{0,159}$/; + +function record(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new DatasetGatewayContractError(`${label}: ожидался объект`); + } + return value as Record; +} + +function array(value: unknown, label: string): unknown[] { + if (!Array.isArray(value)) { + throw new DatasetGatewayContractError(`${label}: ожидался массив`); + } + return value; +} + +function string(value: unknown, label: string, safe = false): string { + if ( + typeof value !== "string" + || !value + || (safe && !SAFE_ID.test(value)) + ) { + throw new DatasetGatewayContractError(`${label}: некорректная строка`); + } + return value; +} + +function strings(value: unknown, label: string): string[] { + return array(value, label).map((item, index) => + string(item, `${label}[${index}]`, true) + ); +} + +function displayStrings(value: unknown, label: string): string[] { + return array(value, label).map((item, index) => + string(item, `${label}[${index}]`) + ); +} + +function boolean(value: unknown, label: string): boolean { + if (typeof value !== "boolean") { + throw new DatasetGatewayContractError(`${label}: ожидался boolean`); + } + return value; +} + +function number(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new DatasetGatewayContractError(`${label}: некорректное число`); + } + return value; +} + +export function parseDatasetGatewayCatalog( + value: unknown, +): DatasetGatewayCatalog { + const source = record(value, "Dataset Gateway"); + if ( + source.schema_version !== "missioncore.dataset-gateway-catalog/v1" + || source.access !== "read-only" + ) { + throw new DatasetGatewayContractError("Dataset Gateway contract несовместим"); + } + const storage = record(source.storage, "storage"); + const storageStatus = storage.status; + if (storageStatus !== "ready" && storageStatus !== "blocked-storage-policy") { + throw new DatasetGatewayContractError("storage.status: неизвестное значение"); + } + if (storage.path_exposed !== false) { + throw new DatasetGatewayContractError("Dataset Gateway раскрыл локальный путь"); + } + const sources = array(source.sources, "sources"); + if (sources.length !== 1) { + throw new DatasetGatewayContractError("Ожидался один первичный dataset source"); + } + const dataset = record(sources[0], "sources[0]"); + const download = record(dataset.download, "source.download"); + const admission = record(dataset.admission, "source.admission"); + if (download.automatic !== false) { + throw new DatasetGatewayContractError("Большой dataset нельзя загружать автоматически"); + } + const admissionStatus = admission.status; + if ( + admissionStatus !== "ready-for-download" + && admissionStatus !== "blocked-storage-policy" + ) { + throw new DatasetGatewayContractError("source admission status неизвестен"); + } + const representations = array( + source.representations, + "representations", + ).map((value, index) => { + const item = record(value, `representations[${index}]`); + const id = item.id; + if ( + id !== "native-scan" + && id !== "normalized-scan" + && id !== "rolling-local-map" + ) { + throw new DatasetGatewayContractError("Неизвестная LiDAR representation"); + } + const normalizedId: DatasetRepresentationId = id; + return { + id: normalizedId, + title: string(item.title, "representation.title"), + purpose: string(item.purpose, "representation.purpose", true), + accumulation: boolean(item.accumulation, "representation.accumulation"), + }; + }); + const pipeline = array(source.pipeline, "pipeline").map((value, index) => { + const item = record(value, `pipeline[${index}]`); + return { + stage: string(item.stage, "pipeline.stage", true), + requires: strings(item.requires, "pipeline.requires"), + produces: string(item.produces, "pipeline.produces", true), + }; + }); + const inputs = array(source.known_inputs, "known_inputs"); + const currentInput = record(inputs[0], "known_inputs[0]"); + if ( + currentInput.representation !== "vendor-mapped-increment" + || currentInput.native_scan !== false + || currentInput.per_point_time !== false + || currentInput.ring_or_line !== false + || currentInput.admitted_for_patchworkpp !== false + ) { + throw new DatasetGatewayContractError("Vendor-map boundary завышен"); + } + if (dataset.frame_semantics !== "one-lidar-revolution") { + throw new DatasetGatewayContractError("GOOSE frame semantics несовместима"); + } + return { + storage: { + configured: boolean(storage.configured, "storage.configured"), + admitted: boolean(storage.admitted, "storage.admitted"), + status: storageStatus, + requiredWindowsRoot: string( + storage.required_windows_root, + "storage.required_windows_root", + ), + requiredWslRoot: string(storage.required_wsl_root, "storage.required_wsl_root"), + }, + source: { + sourceId: string(dataset.source_id, "source_id", true), + displayName: string(dataset.display_name, "display_name"), + role: string(dataset.role, "role", true), + license: string(dataset.license, "license"), + format: string(dataset.format, "format", true), + frameSemantics: "one-lidar-revolution", + platforms: displayStrings(dataset.platforms, "platforms"), + superclasses: strings(dataset.superclasses, "superclasses"), + validationArchiveGb: number( + download.validation_archive_gb, + "validation_archive_gb", + ), + admissionStatus, + }, + representations, + pipeline, + currentInput: { + representation: "vendor-mapped-increment", + nativeScan: false, + perPointTime: false, + ringOrLine: false, + admittedForPatchworkpp: false, + reason: string(currentInput.reason, "known_inputs.reason", true), + }, + nextAction: string(source.next_action, "next_action", true), + }; +} + +async function responseJson(response: Response): Promise { + if (!response.ok) { + throw new Error(`Dataset Gateway HTTP ${response.status}`); + } + return response.json(); +} + +export async function fetchDatasetGatewayCatalog( + options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {}, +): Promise { + const fetcher = options.fetcher ?? fetch; + const response = await fetcher("/api/v1/lidar/dataset-gateway", { + method: "GET", + headers: { Accept: "application/json" }, + signal: options.signal, + }); + return parseDatasetGatewayCatalog(await responseJson(response)); +} diff --git a/apps/control-station/src/styles/responsive.css b/apps/control-station/src/styles/responsive.css index 459c1b3..45ecd11 100644 --- a/apps/control-station/src/styles/responsive.css +++ b/apps/control-station/src/styles/responsive.css @@ -137,6 +137,17 @@ } @media (max-width: 760px) { + .dataset-gateway__representations, + .dataset-gateway__grid, + .dataset-gateway__footer { + grid-template-columns: 1fr; + } + + .dataset-gateway__representations > div + div { + border-top: 1px solid rgb(255 255 255 / 0.07); + border-left: 0; + } + .control-station .nodedc-header__profile-button { display: none; } diff --git a/apps/control-station/src/styles/workspaces.css b/apps/control-station/src/styles/workspaces.css index 550e19c..6784dba 100644 --- a/apps/control-station/src/styles/workspaces.css +++ b/apps/control-station/src/styles/workspaces.css @@ -807,6 +807,144 @@ gap: 1rem; } +.dataset-gateway { + display: grid; + gap: 1rem; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--nodedc-accent) 24%, transparent); + background: + radial-gradient(circle at 10% 0%, color-mix(in srgb, var(--nodedc-accent) 10%, transparent), transparent 32%), + rgb(255 255 255 / 0.025); +} + +.dataset-gateway__heading p, +.dataset-gateway__grid p, +.dataset-gateway__pending { + margin: 0.4rem 0 0; + color: var(--nodedc-text-muted); + font-size: 0.68rem; + line-height: 1.55; +} + +.dataset-gateway__representations { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + overflow: hidden; + border: 1px solid rgb(255 255 255 / 0.07); + border-radius: 0.9rem; +} + +.dataset-gateway__representations > div { + position: relative; + display: grid; + min-height: 7.4rem; + align-content: center; + gap: 0.3rem; + padding: 1rem 1.1rem 1rem 3.6rem; + background: rgb(255 255 255 / 0.025); +} + +.dataset-gateway__representations > div + div { + border-left: 1px solid rgb(255 255 255 / 0.07); +} + +.dataset-gateway__representations > div > span { + position: absolute; + top: 1rem; + left: 1rem; + color: var(--nodedc-accent); + font-size: 0.62rem; + letter-spacing: 0.14em; +} + +.dataset-gateway__representations strong, +.dataset-gateway__representations small, +.dataset-gateway__representations em { + display: block; +} + +.dataset-gateway__representations strong { + color: var(--nodedc-text-primary); + font-size: 0.76rem; +} + +.dataset-gateway__representations small, +.dataset-gateway__representations em { + color: var(--nodedc-text-muted); + font-size: 0.58rem; + line-height: 1.35; +} + +.dataset-gateway__representations em { + color: color-mix(in srgb, var(--nodedc-accent) 72%, white); + font-style: normal; +} + +.dataset-gateway__grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.7rem; +} + +.dataset-gateway__grid > section { + min-width: 0; + padding: 1rem; + border: 1px solid rgb(255 255 255 / 0.07); + border-radius: 0.9rem; + background: rgb(255 255 255 / 0.02); +} + +.dataset-gateway__grid > section[data-warning="true"] { + border-color: rgb(255 181 71 / 0.2); +} + +.dataset-gateway__grid h3 { + margin: 0.28rem 0 0; + color: var(--nodedc-text-primary); + font-size: 1rem; +} + +.dataset-gateway__facts { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-top: 0.8rem; +} + +.dataset-gateway__facts span { + padding: 0.32rem 0.5rem; + border-radius: 999px; + background: rgb(255 255 255 / 0.045); + color: var(--nodedc-text-secondary); + font-size: 0.56rem; +} + +.dataset-gateway__footer { + display: grid; + grid-template-columns: minmax(0, 0.8fr) minmax(0, 1.2fr); + gap: 1rem; + padding-top: 0.85rem; + border-top: 1px solid rgb(255 255 255 / 0.07); +} + +.dataset-gateway__footer > div { + display: grid; + gap: 0.2rem; + min-width: 0; +} + +.dataset-gateway__footer span, +.dataset-gateway__footer small { + color: var(--nodedc-text-muted); + font-size: 0.58rem; +} + +.dataset-gateway__footer strong { + overflow-wrap: anywhere; + color: var(--nodedc-text-primary); + font-size: 0.68rem; +} + .lidar-quality-message { display: grid; min-height: 12rem; diff --git a/apps/control-station/src/workspaces/DatasetGatewayPanel.tsx b/apps/control-station/src/workspaces/DatasetGatewayPanel.tsx new file mode 100644 index 0000000..7809f49 --- /dev/null +++ b/apps/control-station/src/workspaces/DatasetGatewayPanel.tsx @@ -0,0 +1,133 @@ +import { useEffect, useState } from "react"; +import { GlassSurface, StatusBadge } from "@nodedc/ui-react"; + +import { + fetchDatasetGatewayCatalog, + type DatasetGatewayCatalog, +} from "../core/lidar/datasetGateway"; + +const representationLabels: Record = { + "native-scan": "Один оборот / скан", + "normalized-scan": "Deskew + bounded cleanup", + "rolling-local-map": "Pose + TTL + voxel map", +}; + +export function DatasetGatewayPanel() { + const [catalog, setCatalog] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + const controller = new AbortController(); + void fetchDatasetGatewayCatalog({ signal: controller.signal }) + .then((value) => { + if (!controller.signal.aborted) setCatalog(value); + }) + .catch((loadError: unknown) => { + if (controller.signal.aborted) return; + setError( + loadError instanceof Error + ? loadError.message + : "Dataset Gateway недоступен", + ); + }); + return () => controller.abort(); + }, []); + + return ( + +
+
+ DATASET GATEWAY · S0 +

Три разных LiDAR-продукта

+

+ Кольцевой одиночный скан, очищенный sensor-frame и накопленная карта + больше не считаются одним облаком. +

+
+ + {error + ? "Gateway недоступен" + : catalog?.storage.admitted + ? "D: допущен" + : "Ожидает D:"} + +
+ + {catalog ? ( + <> +
+ {catalog.representations.map((representation, index) => ( +
+ 0{index + 1} + + {representationLabels[representation.id] ?? representation.title} + + {representation.title} + + {representation.accumulation + ? "накопление включено явно" + : "без накопления"} + +
+ ))} +
+ +
+
+ ПЕРВЫЙ BASELINE +

{catalog.source.displayName}

+

+ Один оборот VLS-128 в SemanticKITTI XYZI + point-wise semantic и + instance labels. Это то самое разреженное кольцевое облако, + которое корректно сравнивать с алгоритмами. +

+
+ {catalog.source.platforms.join(" · ")} + {catalog.source.superclasses.length} superclasses + val {catalog.source.validationArchiveGb} ГБ + {catalog.source.license} +
+
+
+ ТЕКУЩИЙ DEVICE INPUT +

Mapped feed ≠ raw scan

+

+ Внешний MQTT содержит vendor-mapped increment после LIO. В нём + нет per-point time и line/ring, поэтому из него нельзя честно + восстановить один исходный скан или выполнить deskew. +

+
+ Patchwork++: diagnostic only + rolling map: возможно + raw reconstruction: невозможно +
+
+
+ +
+
+ Storage gate + {catalog.storage.requiredWindowsRoot} + {catalog.storage.requiredWslRoot} +
+
+ Следующий исполнимый шаг + + {catalog.storage.admitted + ? "Скачать GOOSE validation и импортировать первый кадр" + : "Подключить Dataset Root на D: worker"} + + Автозагрузка 3.3 ГБ намеренно запрещена +
+
+ + ) : ( +

+ {error ?? "Читаем входные контракты Dataset Gateway…"} +

+ )} + + ); +} diff --git a/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx b/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx index c5047f3..70b5406 100644 --- a/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx +++ b/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx @@ -26,6 +26,7 @@ import { LidarGroundPointCloud, type LidarGroundViewMode, } from "./LidarGroundPointCloud"; +import { DatasetGatewayPanel } from "./DatasetGatewayPanel"; function formatNumber(value: number | null, digits = 1): string { if (value === null) return "—"; @@ -220,6 +221,8 @@ export function LidarQualityWorkspace({ Только проверенные replay-артефакты + + {loading && !detail ? ( Проверка evidence diff --git a/apps/control-station/test/datasetGateway.test.mjs b/apps/control-station/test/datasetGateway.test.mjs new file mode 100644 index 0000000..dead629 --- /dev/null +++ b/apps/control-station/test/datasetGateway.test.mjs @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import { after, before, test } from "node:test"; + +import { createServer } from "vite"; + +let server; +let parseDatasetGatewayCatalog; +let fetchDatasetGatewayCatalog; +let DatasetGatewayContractError; + +before(async () => { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ + parseDatasetGatewayCatalog, + fetchDatasetGatewayCatalog, + DatasetGatewayContractError, + } = await server.ssrLoadModule("/src/core/lidar/datasetGateway.ts")); +}); + +after(async () => { + await server?.close(); +}); + +function catalog(overrides = {}) { + return { + schema_version: "missioncore.dataset-gateway-catalog/v1", + access: "read-only", + storage: { + configured: false, + required_windows_root: "D:\\NDC_MISSIONCORE\\datasets", + required_wsl_root: "/mnt/d/NDC_MISSIONCORE/datasets", + admitted: false, + status: "blocked-storage-policy", + path_exposed: false, + }, + sources: [{ + source_id: "goose-3d/v2025-08-22", + display_name: "GOOSE 3D", + role: "primary-offroad-semantic-baseline", + license: "CC-BY-SA-4.0", + format: "semantickitti-xyzi-label", + frame_semantics: "one-lidar-revolution", + platforms: ["MuCAR-3", "ALICE", "Spot"], + annotations: ["semantic-point", "instance-point"], + superclasses: ["natural-ground", "obstacle"], + download: { + automatic: false, + reason: "operator-admitted-large-artifact-only", + validation_archive_gb: 3.3, + }, + admission: { + status: "blocked-storage-policy", + }, + }], + representations: [ + { + id: "native-scan", + title: "Одиночный исходный скан", + purpose: "dataset-ground-truth-and-sensor-domain", + accumulation: false, + }, + { + id: "normalized-scan", + title: "Нормализованный sensor-frame скан", + purpose: "deskew-filter-inference-and-algorithm-comparison", + accumulation: false, + }, + { + id: "rolling-local-map", + title: "Накопленная локальная карта", + purpose: "stable-operator-view-and-local-planning", + accumulation: true, + }, + ], + pipeline: [{ + stage: "deskew", + requires: ["per-point-time", "imu-or-odometry"], + produces: "normalized-scan", + }], + known_inputs: [{ + source_id: "current-recorded-lidar/vendor-map", + representation: "vendor-mapped-increment", + native_scan: false, + per_point_time: false, + ring_or_line: false, + admitted_for_patchworkpp: false, + reason: "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan", + }], + next_action: "configure-dataset-root-on-worker-d", + ...overrides, + }; +} + +test("parses three distinct LiDAR representations and current-input boundary", () => { + const parsed = parseDatasetGatewayCatalog(catalog()); + assert.deepEqual( + parsed.representations.map((item) => item.id), + ["native-scan", "normalized-scan", "rolling-local-map"], + ); + assert.equal(parsed.representations[2].accumulation, true); + assert.equal(parsed.currentInput.admittedForPatchworkpp, false); + assert.equal(parsed.source.frameSemantics, "one-lidar-revolution"); +}); + +test("rejects path exposure and upgraded K1 semantics", () => { + const exposed = catalog(); + exposed.storage.path_exposed = true; + assert.throws( + () => parseDatasetGatewayCatalog(exposed), + DatasetGatewayContractError, + ); + + const upgraded = catalog(); + upgraded.known_inputs[0].per_point_time = true; + assert.throws( + () => parseDatasetGatewayCatalog(upgraded), + DatasetGatewayContractError, + ); +}); + +test("fetches the read-only gateway endpoint", async () => { + const calls = []; + const parsed = await fetchDatasetGatewayCatalog({ + fetcher: async (url, init) => { + calls.push({ url, init }); + return new Response(JSON.stringify(catalog()), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }, + }); + assert.equal(calls[0].url, "/api/v1/lidar/dataset-gateway"); + assert.equal(calls[0].init.method, "GET"); + assert.equal(parsed.source.displayName, "GOOSE 3D"); +}); diff --git a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md index b11b829..0a53fc4 100644 --- a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md +++ b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md @@ -1,8 +1,8 @@ # LiDAR worker: product value, evidence boundary and implementation roadmap Date: 2026-07-25 -Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete, -independent labels pending +Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete; +Dataset Gateway S0 implemented, first real GOOSE import pending worker D storage Scope: real scanner records, replay and future live shadow processing Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering @@ -110,7 +110,9 @@ unknown, camera-only velocity is not metric, 374 conflicts remain, and the benchmark is not independent ground truth. The next work must therefore improve the LiDAR-native input and evaluation -surface, not add another visual smoothing pass. +surface, not add another visual smoothing pass. Mission Core will use +independently labeled public datasets before requesting any new manual K1 +annotation. The current K1 evidence stays an unlabeled out-of-domain smoke test. ## 4. Market and stack assessment @@ -236,10 +238,10 @@ quality line. It is not repaired or hidden by replay. distributions in `missioncore.lidar-ground-benchmark/v1`. - [x] Create an immutable eight-frame annotation template in which every point starts as `ignore-unreviewed`; it is explicitly not ground truth. -- [ ] Complete independent human review for ground, curb, low obstacle, - reflection noise and other non-ground points. -- [ ] Measure accepted ground IoU, curb/low-obstacle recall and - reflection-noise rejection against that reviewed generation. +- [ ] Keep the K1 annotation template frozen as an optional later + domain-adaptation asset; do not make manual review the current critical path. +- [ ] Measure accepted ground IoU and obstacle recall first against an admitted + GOOSE native scan and its published point-wise labels. The real diagnostic run is `ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`. @@ -285,16 +287,43 @@ geometry is not exported. The measured 1.27 m estimate can be reproduced as a named diagnostic profile; tuning that offset until Patchwork++ merely looks plausible would still invalidate the comparison. -The independent-label exit remains open. It can be closed by reviewing the -content-bound template -`ground-annotation-template-12e12eab0756c14f5e06adcaf13189e93188df9bb6d31224cb9b7d566ec15b18`, -or by acquiring an admitted raw sensor scan with known physical sensor height -and then producing a new benchmark generation. +The K1-specific independent-label exit remains open, but it is no longer the +next step. The Dataset Gateway provides a labeled public-domain evaluation +path. K1 manual review or a new real vehicle dataset is reserved for later +domain adaptation after a public baseline proves that the pipeline and metric +harness work. + +### L2.5 — Dataset Gateway — S0 complete, real import pending + +- [x] Define separate `native-scan`, `normalized-scan` and + `rolling-local-map` representations. +- [x] Add a lossless GOOSE/SemanticKITTI XYZI + packed semantic/instance label + reader with point-alignment and finite-value gates. +- [x] Publish the read-only Dataset Gateway contract in React. +- [x] Refuse to relabel current `lio_pcl` as a native scan or to invent missing + per-point time and line/ring fields. +- [x] Require operator-admitted storage under + `D:\NDC_MISSIONCORE\datasets`. +- [ ] Configure the worker dataset root and record disk/resource baseline. +- [ ] Download only the 3.3 GB GOOSE validation archive first and record its + hash/license/provenance. +- [ ] Show one real labeled revolution in React with native remission and + ground-truth superclass coloring. +- [ ] Admit an explicit frame/mounting profile before normalization. +- [ ] Run current ground and Patchwork++ against GOOSE ground truth. +- [ ] Add named range/FOV/density/noise/dropout degradation profiles without + overwriting the native frame. + +The architectural contract and run sequence are fixed in +`docs/14_LIDAR_DATASET_GATEWAY.md` and ADR 0021. GOOSE is first because its +published 3D format is one LiDAR revolution with point-wise semantic and +instance labels in off-road environments. RELLIS-3D remains the second-source +cross-check after the GOOSE harness is stable. ### L3 — LiDAR-native 3D detection -- [ ] Freeze independent 3D annotations covering people, vehicles, cyclists, - stroller groups, vegetation and confusing static structures. +- [ ] Establish the public-dataset baseline first; freeze K1-specific 3D + annotations only when a measured domain gap justifies them. - [ ] Run NVIDIA PointPillars through the existing external worker/Triton seam. - [ ] Treat pretrained output as a baseline, not an accepted product model. - [ ] Measure class precision/recall, center/range/yaw error, distance-bucket @@ -358,6 +387,8 @@ The near-term value is not a prettier point cloud: - hardware selection becomes evidence-driven: a future vehicle LiDAR is accepted by its timing/fields/profile, not by vendor marketing. -The highest-value immediate work is L1, followed by L2 and L3. Nvblox and -alternative SLAM are useful, but starting them before their input gates would -produce attractive demos with unqualified geometry. +The highest-value immediate work is the worker-side GOOSE validation import, +the first labeled native scan in React and an accuracy-bearing ground A/B. +LiDAR-native detection follows on the same gateway. Nvblox and alternative +SLAM remain later because their timing, pose and scan-geometry gates are not yet +satisfied. diff --git a/docs/14_LIDAR_DATASET_GATEWAY.md b/docs/14_LIDAR_DATASET_GATEWAY.md new file mode 100644 index 0000000..59a5cbb --- /dev/null +++ b/docs/14_LIDAR_DATASET_GATEWAY.md @@ -0,0 +1,140 @@ +# LiDAR Dataset Gateway + +## Product value + +The Dataset Gateway gives Mission Core a repeatable perception laboratory +before the production vehicle and its final sensor installation exist. It +separates four questions that were previously mixed together: + +- whether the transport preserved the sensor evidence; +- whether preprocessing produces a valid one-scan perception input; +- whether an algorithm is accurate against independent labels; +- whether several scans form a stable local map for an operator or planner. + +This prevents tuning an algorithm until a visually dense vendor map merely +looks plausible. It also keeps work reusable across Gazebo, Unreal, public +datasets and future real onboard sensors. + +## Current S0 slice + +Implemented now: + +- `missioncore.dataset-gateway-catalog/v1`, exposed read-only at + `GET /api/v1/lidar/dataset-gateway`; +- explicit `native-scan`, `normalized-scan` and `rolling-local-map` + representations; +- a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI + and packed uint32 semantic/instance labels; +- count, finite-value and maximum-point safety gates; +- immutable point-aligned arrays; +- a fail-closed K1 `lio_pcl` boundary; +- worker storage admission for `D:\NDC_MISSIONCORE\datasets` and + `/mnt/d/NDC_MISSIONCORE/datasets`; +- a visible Dataset Gateway panel in **Данные → Качество LiDAR**. + +Not implemented in S0: + +- no automatic 3.3 GB validation archive download; +- no implicit coordinate conversion; +- no fake ring/timestamp reconstruction for K1 MQTT evidence; +- no model training or production promotion; +- no rolling-map implementation yet. + +## Why public recordings look different + +GOOSE stores one VLS-128 revolution per annotated `.bin` file. A rotating +multi-channel sensor produces discrete scan lines, so a single sensor-frame +view looks like sparse rings. + +The current field review is explicitly an accumulated map-frame window. It +combines many source publications after pose registration. This fills surfaces +and hides the original scan pattern. The external K1 stream is also already a +post-LIO/modeling product and lacks the raw driver fields needed to reconstruct +an original scan. + +Livox sensors additionally use a scan pattern that differs from classic fixed +vertical channels. Time integration therefore changes their visual density in +a different way. “Ring-like” is a sensor geometry property, not a universal +quality target. + +## Canonical processing profiles + +### P0 — native evidence + +Required: + +- source ID and immutable frame ID; +- XYZ and the original return/remission/intensity field; +- semantic and instance labels when present; +- calibration/mounting/timing evidence as separate metadata; +- no accumulation and no hidden world transform. + +Output: `native-scan`. + +### P1 — normalized perception scan + +Ordered operations: + +1. decode and apply only evidenced factory calibration; +2. assign an explicit sensor coordinate frame; +3. deskew when per-point time and synchronized motion are available; +4. apply bounded range and field-of-view policy; +5. remove the vehicle/self mask; +6. apply named outlier and voxel policies; +7. retain a reversible index/provenance map to the native frame. + +Output: `normalized-scan`. + +### P2 — inference + +Ground, semantic and object providers consume P1. Patchwork++ belongs here. It +does not own P0/P1 or P3. + +Output: point-aligned predictions and reproducible metrics against labels. + +### P3 — rolling local map + +Ordered operations: + +1. bind each normalized scan to an evidenced pose; +2. transform to `odom` or a declared local-map frame; +3. deduplicate with a named voxel policy; +4. expire points by TTL or travelled distance; +5. keep dynamic points short-lived or track them separately; +6. publish bounded map state and its contributing frame identities. + +Output: `rolling-local-map`. + +This is the stage that should stop static geometry from “jumping”. Deskew +reduces within-scan motion distortion; registration stabilizes scans across +time; TTL/dynamic filtering prevents stale ghosts. + +## First dataset sequence + +1. Configure `MISSIONCORE_DATASET_ROOT=/mnt/d/NDC_MISSIONCORE/datasets` on the + Windows/WSL worker. +2. Verify free space and record archive size/hash/license. +3. Download only the GOOSE 3D validation archive first (published size 3.3 GB). +4. Import one labeled frame and expose it in React as `native-scan`. +5. Show native remission and ground-truth superclass coloring. +6. Add a declared GOOSE frame/mounting profile and produce `normalized-scan`. +7. Run current ground heuristic and Patchwork++ against independent labels. +8. Add sensor-degradation profiles for range, FOV, density, noise and dropout. +9. Only after the one-frame contract passes, expand to the validation split and + add a rolling-map sequence with localization evidence. + +## Acceptance checklist + +- [x] Representations cannot be silently interchanged. +- [x] Large artifacts require operator-admitted D-only storage. +- [x] GOOSE XYZI and labels remain point aligned. +- [x] Invalid length and non-finite frames fail closed. +- [x] K1 mapped increments cannot claim raw-scan fields. +- [x] React exposes the architectural truth before dataset bytes exist. +- [ ] Worker D root configured. +- [ ] GOOSE validation archive hash recorded. +- [ ] First real labeled frame visible in React. +- [ ] Coordinate and mounting profile admitted. +- [ ] Patchwork++ accuracy measured against ground truth. +- [ ] Sensor-degradation matrix qualified. +- [ ] Rolling local map with pose/TTL/dynamic policy qualified. diff --git a/docs/adr/0021-dataset-gateway-representation-boundary.md b/docs/adr/0021-dataset-gateway-representation-boundary.md new file mode 100644 index 0000000..31556b1 --- /dev/null +++ b/docs/adr/0021-dataset-gateway-representation-boundary.md @@ -0,0 +1,90 @@ +# ADR 0021: separate native scans, normalized scans and rolling local maps + +Status: accepted +Date: 2026-07-25 + +## Context + +Public autonomous-driving and field-robotics dataset viewers commonly display +one LiDAR revolution. Fixed-channel rotating sensors therefore produce the +familiar sparse rings. The current XGRIDS K1 MQTT `lio_pcl` evidence is a +different product: firmware evidence places it after LIO/modeling, and the +field-review UI accumulates multiple already registered increments in the map +frame. + +Point count alone does not make these representations comparable. A cloud can +be sparse per publication and still look dense after several seconds of pose +registration. Conversely, voxel downsampling does not restore timing, scan +lines or raw sensor geometry that the source no longer carries. + +Patchwork++ is a ground classifier. It does not decode sensor packets, deskew +motion distortion, estimate pose, stabilize a rolling map or remove ghosts +from stale/dynamic observations. A successful Patchwork++ call against +`lio_pcl` remains diagnostic and does not repair the input domain. + +## Decision + +Mission Core defines three non-interchangeable LiDAR products: + +1. `native-scan`: one losslessly decoded source scan/frame with its native + point-aligned fields and labels. It is never accumulated. +2. `normalized-scan`: one sensor-frame scan after an explicit transform, + deskew and bounded cleanup profile. Every transformation retains source + identity and point alignment. +3. `rolling-local-map`: normalized scans registered by pose into a bounded + local map with explicit TTL, voxel deduplication and dynamic-point policy. + +The Dataset Gateway is the first producer of this contract. GOOSE 3D is the +first admitted source because it publishes off-road point-wise semantic and +instance labels in SemanticKITTI-compatible `XYZI + uint32 label` files. Its +annotated point-cloud file represents one LiDAR revolution. + +The gateway: + +- preserves the native GOOSE frame before adaptation; +- never transforms labels independently of their points; +- does not assume a coordinate convention, mounting transform or sensor height + unless source metadata supplies it; +- refuses automatic downloads of large archives; +- admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror; +- never promotes K1 `lio_pcl` to `native-scan`. + +The normalized pipeline is: + +```text +native packet/source frame + -> decode + calibration + -> per-point-time deskew + -> range/self/outlier/voxel policy + -> normalized-scan + -> ground/object inference + -> pose registration + TTL + voxel deduplication + -> rolling-local-map +``` + +Deskew is conditional: it requires per-point time plus synchronized IMU or +odometry. If those fields are missing, the gateway reports the stage as +unavailable rather than inventing timestamps. + +## Consequences + +- The UI must label accumulated K1 evidence as a map product, not a scan. +- Dataset and device inputs can share downstream algorithms only after their + normalized contracts match. +- Sensor adaptation may change range, FOV, point density, noise and dropout for + robustness experiments, but it cannot recreate lost timestamps, occlusions + or material response. +- Patchwork++ becomes eligible for a real quality gate only on a sensor-centric + scan with declared scan geometry and physical mounting height plus + independent labels. +- Stable operator visualization is owned by rolling-map policy, not by the + ground classifier. + +## Primary references + +- [GOOSE dataset structure](https://goose-dataset.de/docs/dataset-structure/) +- [GOOSE setup and archive sizes](https://goose-dataset.de/docs/setup/) +- [GOOSE 3D challenge ontology](https://goose-dataset.de/docs/3d-semantic-segmentation-challenge/) +- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2) +- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox) +- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html) diff --git a/src/k1link/datasets/__init__.py b/src/k1link/datasets/__init__.py new file mode 100644 index 0000000..0d25e08 --- /dev/null +++ b/src/k1link/datasets/__init__.py @@ -0,0 +1,17 @@ +"""Vendor-neutral dataset ingress for perception qualification.""" + +from k1link.datasets.gateway import ( + DATASET_GATEWAY_CATALOG_SCHEMA, + DatasetFrameError, + DatasetPointFrame, + dataset_gateway_catalog, + read_semantic_kitti_frame, +) + +__all__ = [ + "DATASET_GATEWAY_CATALOG_SCHEMA", + "DatasetFrameError", + "DatasetPointFrame", + "dataset_gateway_catalog", + "read_semantic_kitti_frame", +] diff --git a/src/k1link/datasets/gateway.py b/src/k1link/datasets/gateway.py new file mode 100644 index 0000000..2823b1f --- /dev/null +++ b/src/k1link/datasets/gateway.py @@ -0,0 +1,227 @@ +"""Dataset-first LiDAR ingress with explicit representation boundaries. + +The gateway never treats a registered map increment as a native sensor scan. +It first preserves one source frame and its labels. Normalization and rolling +map construction are separate, provenance-bearing products. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path, PureWindowsPath +from typing import Final, Literal + +import numpy as np +import numpy.typing as npt + +DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v1" +DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT" +Representation = Literal["native-scan", "normalized-scan", "rolling-local-map"] + + +class DatasetFrameError(ValueError): + """A dataset frame cannot satisfy its declared lossless source contract.""" + + +@dataclass(frozen=True) +class DatasetPointFrame: + """One SemanticKITTI-compatible labeled LiDAR frame. + + GOOSE publishes one LiDAR revolution per ``.bin`` frame. Coordinates and + remission stay source-native here; no vehicle transform or accumulation is + silently applied. + """ + + points_xyz_m: npt.NDArray[np.float32] + remission: npt.NDArray[np.float32] + semantic_labels: npt.NDArray[np.uint16] + instance_labels: npt.NDArray[np.uint16] + representation: Representation = "native-scan" + + @property + def point_count(self) -> int: + return int(self.points_xyz_m.shape[0]) + + +def read_semantic_kitti_frame( + point_path: Path, + label_path: Path, + *, + maximum_points: int = 2_000_000, +) -> DatasetPointFrame: + """Read one GOOSE/SemanticKITTI XYZI + packed-label frame losslessly.""" + + if maximum_points <= 0: + raise DatasetFrameError("maximum_points must be positive") + try: + point_size = point_path.stat().st_size + label_size = label_path.stat().st_size + except OSError as exc: + raise DatasetFrameError("dataset point or label file is unavailable") from exc + if point_size == 0 or point_size % 16: + raise DatasetFrameError("point frame must contain little-endian float32 XYZI tuples") + if label_size % 4: + raise DatasetFrameError("label frame must contain packed little-endian uint32 values") + point_count = point_size // 16 + if point_count > maximum_points: + raise DatasetFrameError("dataset point frame exceeds the configured safety limit") + if label_size // 4 != point_count: + raise DatasetFrameError("point and label counts do not match") + + try: + xyzi = np.fromfile(point_path, dtype="> np.uint32(16), dtype=np.uint16) + for values in (points, remission, semantic, instance): + values.setflags(write=False) + return DatasetPointFrame(points, remission, semantic, instance) + + +def _configured_dataset_root() -> Path | None: + raw = os.environ.get(DATASET_ROOT_ENV, "").strip() + return Path(raw).expanduser().absolute() if raw else None + + +def _is_worker_d_storage(root: Path | None) -> bool: + if root is None: + return False + raw = str(root) + windows = PureWindowsPath(raw) + if windows.drive.upper() == "D:": + return True + normalized = raw.replace("\\", "/").rstrip("/").lower() + return normalized == "/mnt/d/ndc_missioncore/datasets" or normalized.startswith( + "/mnt/d/ndc_missioncore/datasets/" + ) + + +def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, object]: + """Return the path-free, read-only ingress plan and current admission state.""" + + root = dataset_root if dataset_root is not None else _configured_dataset_root() + storage_admitted = _is_worker_d_storage(root) + return { + "schema_version": DATASET_GATEWAY_CATALOG_SCHEMA, + "access": "read-only", + "storage": { + "configured": root is not None, + "required_windows_root": r"D:\NDC_MISSIONCORE\datasets", + "required_wsl_root": "/mnt/d/NDC_MISSIONCORE/datasets", + "admitted": storage_admitted, + "status": "ready" if storage_admitted else "blocked-storage-policy", + "path_exposed": False, + }, + "sources": [ + { + "source_id": "goose-3d/v2025-08-22", + "display_name": "GOOSE 3D", + "role": "primary-offroad-semantic-baseline", + "license": "CC-BY-SA-4.0", + "format": "semantickitti-xyzi-label", + "frame_semantics": "one-lidar-revolution", + "platforms": ["MuCAR-3", "ALICE", "Spot"], + "annotations": ["semantic-point", "instance-point"], + "superclasses": [ + "other", + "artificial-structures", + "artificial-ground", + "natural-ground", + "obstacle", + "vehicle", + "vegetation", + "human", + "sky", + ], + "download": { + "automatic": False, + "reason": "operator-admitted-large-artifact-only", + "training_archive_gb": 27.0, + "validation_archive_gb": 3.3, + "test_archive_gb": 3.3, + }, + "admission": { + "status": ( + "ready-for-download" if storage_admitted else "blocked-storage-policy" + ), + "native_scan": "ready-after-download", + "normalized_scan": "requires-explicit-frame-and-mounting-contract", + "rolling_local_map": "requires-pose-timing-and-map-policy", + }, + } + ], + "representations": [ + { + "id": "native-scan", + "title": "Одиночный исходный скан", + "retains": ["xyz", "remission", "semantic-label", "instance-label"], + "purpose": "dataset-ground-truth-and-sensor-domain", + "accumulation": False, + }, + { + "id": "normalized-scan", + "title": "Нормализованный sensor-frame скан", + "retains": ["source-provenance", "point-alignment", "labels"], + "purpose": "deskew-filter-inference-and-algorithm-comparison", + "accumulation": False, + }, + { + "id": "rolling-local-map", + "title": "Накопленная локальная карта", + "retains": ["source-frame-ids", "pose-provenance", "age"], + "purpose": "stable-operator-view-and-local-planning", + "accumulation": True, + }, + ], + "pipeline": [ + { + "stage": "decode-and-calibrate", + "requires": ["native-packets-or-source-frame", "calibration"], + "produces": "native-scan", + }, + { + "stage": "deskew", + "requires": ["per-point-time", "imu-or-odometry"], + "produces": "normalized-scan", + }, + { + "stage": "bounded-cleanup", + "requires": ["range-policy", "self-mask", "outlier-policy", "voxel-policy"], + "produces": "normalized-scan", + }, + { + "stage": "ground-and-object-inference", + "requires": ["normalized-scan", "declared-sensor-height-and-geometry"], + "produces": "point-aligned-predictions", + }, + { + "stage": "pose-registration-and-rolling-map", + "requires": ["normalized-scan", "pose", "ttl", "voxel-deduplication"], + "produces": "rolling-local-map", + }, + ], + "known_inputs": [ + { + "source_id": "current-recorded-lidar/vendor-map", + "representation": "vendor-mapped-increment", + "native_scan": False, + "per_point_time": False, + "ring_or_line": False, + "admitted_for_patchworkpp": False, + "reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan", + } + ], + "next_action": ( + "download-goose-validation-to-d" + if storage_admitted + else "configure-dataset-root-on-worker-d" + ), + } diff --git a/src/k1link/web/lidar_api.py b/src/k1link/web/lidar_api.py index 7d91522..05a98c4 100644 --- a/src/k1link/web/lidar_api.py +++ b/src/k1link/web/lidar_api.py @@ -20,6 +20,7 @@ from k1link.compute import ( lidar_pack_catalog_item, lidar_pack_detail, ) +from k1link.datasets import dataset_gateway_catalog LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1" LIDAR_GROUND_CATALOG_SCHEMA: Final = "missioncore.lidar-ground-benchmark-catalog/v1" @@ -53,6 +54,10 @@ def build_lidar_router( ) -> APIRouter: router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"]) + @router.get("/dataset-gateway") + def get_dataset_gateway() -> dict[str, object]: + return dataset_gateway_catalog() + @router.get("/replay-packs") def list_lidar_replay_packs( limit: int = Query(default=20, ge=1, le=100), diff --git a/tests/test_dataset_gateway.py b/tests/test_dataset_gateway.py new file mode 100644 index 0000000..3ef6d48 --- /dev/null +++ b/tests/test_dataset_gateway.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +from fastapi import APIRouter +from fastapi.routing import APIRoute + +from k1link.datasets import ( + DatasetFrameError, + dataset_gateway_catalog, + read_semantic_kitti_frame, +) +from k1link.web.lidar_api import build_lidar_router + + +def _endpoint(router: APIRouter, path: str) -> object: + for route in router.routes: + if isinstance(route, APIRoute) and route.path == path and "GET" in route.methods: + return route.endpoint + raise AssertionError(f"GET {path} route is missing") + + +def test_goose_semantickitti_frame_retains_xyzi_and_packed_labels( + tmp_path: Path, +) -> None: + points_path = tmp_path / "frame_vls128.bin" + labels_path = tmp_path / "frame_goose.label" + np.asarray( + [ + [1.0, 2.0, 3.0, 0.25], + [-4.0, 5.0, 0.5, 0.75], + ], + dtype=" None: + points_path = tmp_path / "frame.bin" + labels_path = tmp_path / "frame.label" + np.asarray([[1.0, 2.0, 3.0, 0.25]], dtype=" None: + blocked = dataset_gateway_catalog(tmp_path / "datasets") + admitted = dataset_gateway_catalog(Path("/mnt/d/NDC_MISSIONCORE/datasets")) + + assert blocked["storage"]["status"] == "blocked-storage-policy" # type: ignore[index] + assert admitted["storage"]["status"] == "ready" # type: ignore[index] + assert admitted["next_action"] == "download-goose-validation-to-d" + representations = admitted["representations"] + assert isinstance(representations, list) + assert [item["id"] for item in representations] == [ # type: ignore[index] + "native-scan", + "normalized-scan", + "rolling-local-map", + ] + inputs = admitted["known_inputs"] + assert isinstance(inputs, list) + assert inputs[0]["admitted_for_patchworkpp"] is False # type: ignore[index] + + +def test_dataset_gateway_api_is_read_only_and_path_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATASET_ROOT", "/mnt/d/NDC_MISSIONCORE/datasets") + route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway") + response = route() # type: ignore[operator] + + assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v1" + assert response["access"] == "read-only" + assert response["storage"]["admitted"] is True + assert response["storage"]["path_exposed"] is False + assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace( + response["storage"]["required_wsl_root"], # type: ignore[index] + "", + )