From 951b40c870e3d360c127373bd1fc33a509c9f821 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sat, 25 Jul 2026 14:14:20 +0300 Subject: [PATCH] feat(lidar): admit and benchmark GOOSE baseline --- .../src/core/lidar/datasetGateway.ts | 380 ++++++++++++- .../control-station/src/styles/responsive.css | 9 + .../control-station/src/styles/workspaces.css | 122 ++++ .../workspaces/DatasetGatewayWorkspace.tsx | 244 +++++++- .../src/workspaces/LidarGroundPointCloud.tsx | 41 +- .../test/datasetGateway.test.mjs | 110 +++- docs/14_LIDAR_DATASET_GATEWAY.md | 111 +++- ...dataset-gateway-representation-boundary.md | 18 +- pyproject.toml | 1 + src/k1link/compute/lidar_ground.py | 197 +------ src/k1link/datasets/__init__.py | 22 + src/k1link/datasets/cli.py | 67 +++ src/k1link/datasets/gateway.py | 402 +++++++++++++- src/k1link/datasets/goose_admission.py | 519 ++++++++++++++++++ src/k1link/datasets/goose_benchmark.py | 270 +++++++++ src/k1link/ground_segmentation.py | 238 ++++++++ src/k1link/web/app.py | 9 + src/k1link/web/lidar_api.py | 50 +- tests/test_dataset_gateway.py | 260 ++++++++- tests/test_lidar_ground.py | 42 ++ 20 files changed, 2871 insertions(+), 241 deletions(-) create mode 100644 src/k1link/datasets/cli.py create mode 100644 src/k1link/datasets/goose_admission.py create mode 100644 src/k1link/datasets/goose_benchmark.py create mode 100644 src/k1link/ground_segmentation.py diff --git a/apps/control-station/src/core/lidar/datasetGateway.ts b/apps/control-station/src/core/lidar/datasetGateway.ts index 40eef50..dd2bdb3 100644 --- a/apps/control-station/src/core/lidar/datasetGateway.ts +++ b/apps/control-station/src/core/lidar/datasetGateway.ts @@ -8,6 +8,8 @@ export interface DatasetGatewayCatalog { configured: boolean; admitted: boolean; status: "ready" | "blocked-storage-policy"; + attestation: "worker-manifest" | "local-worker-path" | "none"; + manifestValid: boolean; requiredWindowsRoot: string; requiredWslRoot: string; }; @@ -21,7 +23,31 @@ export interface DatasetGatewayCatalog { platforms: string[]; superclasses: string[]; validationArchiveGb: number; - admissionStatus: "ready-for-download" | "blocked-storage-policy"; + admissionStatus: + | "blocked-storage-policy" + | "ready-for-download" + | "downloading" + | "downloaded" + | "verifying" + | "verified" + | "frame-ready"; + archive: { + bytesTransferred: number; + totalBytes: number; + sizeBytes: number | null; + sha256: string | null; + integrity: string; + vendorChecksumAvailable: false; + } | null; + frame: { + frameId: string; + pointCount: number; + semanticClassCount: number; + groundTruthGroundFraction: number; + previewPointCount: number; + previewSha256: string; + previewAvailable: true; + } | null; }; representations: Array<{ id: DatasetRepresentationId; @@ -45,6 +71,46 @@ export interface DatasetGatewayCatalog { nextAction: string; } +export interface DatasetNativeScanPreview { + sourceId: "goose-3d/v2025-08-22"; + frameId: string; + sourcePointCount: number; + pointCount: number; + pointsXyzM: Array<[number, number, number]>; + remission0To255: number[]; + semanticLabelIds: number[]; + semanticRgb0To255: number[]; + groundTruthGround: number[]; + classes: Array<{ + labelId: number; + className: string; + hex: string; + challengeCategoryId: number; + challengeCategoryName: string; + }>; +} + +export interface DatasetGroundComparison { + sourceId: "goose-3d/v2025-08-22"; + frameId: string; + pointCount: number; + currentGround: number[]; + groundTruthGround: number[]; + evaluated: number[]; + disagreement: number[]; + metrics: { + precision: number; + recall: number; + f1: number; + groundIou: number; + accuracy: number; + artificialGroundRecall: number; + naturalGroundRecall: number; + obstacleNonGroundRecall: number; + }; + latencyMs: number; +} + export class DatasetGatewayContractError extends Error {} type DatasetFetch = ( @@ -105,12 +171,43 @@ function number(value: unknown, label: string): number { return value; } +function nullableNumber(value: unknown, label: string): number | null { + return value === null ? null : number(value, label); +} + +function nullableDigest(value: unknown, label: string): string | null { + if (value === null) return null; + const digest = string(value, label); + if (!/^[a-f0-9]{64}$/.test(digest)) { + throw new DatasetGatewayContractError(`${label}: некорректный SHA-256`); + } + return digest; +} + +function integers( + value: unknown, + label: string, + maximum: number, +): number[] { + return array(value, label).map((item, index) => { + if ( + typeof item !== "number" + || !Number.isInteger(item) + || item < 0 + || item > maximum + ) { + throw new DatasetGatewayContractError(`${label}[${index}]: некорректное число`); + } + return item; + }); +} + export function parseDatasetGatewayCatalog( value: unknown, ): DatasetGatewayCatalog { const source = record(value, "Dataset Gateway"); if ( - source.schema_version !== "missioncore.dataset-gateway-catalog/v1" + source.schema_version !== "missioncore.dataset-gateway-catalog/v2" || source.access !== "read-only" ) { throw new DatasetGatewayContractError("Dataset Gateway contract несовместим"); @@ -123,6 +220,14 @@ export function parseDatasetGatewayCatalog( if (storage.path_exposed !== false) { throw new DatasetGatewayContractError("Dataset Gateway раскрыл локальный путь"); } + const storageAttestation = storage.attestation; + if ( + storageAttestation !== "worker-manifest" + && storageAttestation !== "local-worker-path" + && storageAttestation !== "none" + ) { + throw new DatasetGatewayContractError("storage.attestation: неизвестное значение"); + } const sources = array(source.sources, "sources"); if (sources.length !== 1) { throw new DatasetGatewayContractError("Ожидался один первичный dataset source"); @@ -137,9 +242,72 @@ export function parseDatasetGatewayCatalog( if ( admissionStatus !== "ready-for-download" && admissionStatus !== "blocked-storage-policy" + && admissionStatus !== "downloading" + && admissionStatus !== "downloaded" + && admissionStatus !== "verifying" + && admissionStatus !== "verified" + && admissionStatus !== "frame-ready" ) { throw new DatasetGatewayContractError("source admission status неизвестен"); } + const archive = admission.archive === null + ? null + : record(admission.archive, "source.admission.archive"); + const archiveValue = archive + ? { + bytesTransferred: number( + archive.bytes_transferred, + "archive.bytes_transferred", + ), + totalBytes: number(archive.total_bytes, "archive.total_bytes"), + sizeBytes: nullableNumber(archive.size_bytes, "archive.size_bytes"), + sha256: nullableDigest(archive.sha256, "archive.sha256"), + integrity: string(archive.integrity, "archive.integrity", true), + vendorChecksumAvailable: (() => { + if (archive.vendor_checksum_available !== false) { + throw new DatasetGatewayContractError( + "archive.vendor_checksum_available: ожидался false", + ); + } + return false as const; + })(), + } + : null; + const admittedFrame = admission.frame === null + ? null + : record(admission.frame, "source.admission.frame"); + const frameValue = admittedFrame + ? { + frameId: string(admittedFrame.frame_id, "frame.frame_id", true), + pointCount: number(admittedFrame.point_count, "frame.point_count"), + semanticClassCount: number( + admittedFrame.semantic_class_count, + "frame.semantic_class_count", + ), + groundTruthGroundFraction: number( + admittedFrame.ground_truth_ground_fraction, + "frame.ground_truth_ground_fraction", + ), + previewPointCount: number( + admittedFrame.preview_point_count, + "frame.preview_point_count", + ), + previewSha256: nullableDigest( + admittedFrame.preview_sha256, + "frame.preview_sha256", + ) ?? (() => { + throw new DatasetGatewayContractError("frame.preview_sha256 отсутствует"); + })(), + previewAvailable: (() => { + if (admittedFrame.preview_available !== true) { + throw new DatasetGatewayContractError( + "frame.preview_available: ожидался true", + ); + } + return true as const; + })(), + } + : null; const representations = array( source.representations, "representations", @@ -188,6 +356,8 @@ export function parseDatasetGatewayCatalog( configured: boolean(storage.configured, "storage.configured"), admitted: boolean(storage.admitted, "storage.admitted"), status: storageStatus, + attestation: storageAttestation, + manifestValid: boolean(storage.manifest_valid, "storage.manifest_valid"), requiredWindowsRoot: string( storage.required_windows_root, "storage.required_windows_root", @@ -208,6 +378,8 @@ export function parseDatasetGatewayCatalog( "validation_archive_gb", ), admissionStatus, + archive: archiveValue, + frame: frameValue, }, representations, pipeline, @@ -223,6 +395,183 @@ export function parseDatasetGatewayCatalog( }; } +export function parseDatasetNativeScanPreview( + value: unknown, +): DatasetNativeScanPreview { + const source = record(value, "Dataset preview"); + if ( + source.schema_version !== "missioncore.dataset-native-scan-preview/v1" + || source.source_id !== "goose-3d/v2025-08-22" + || source.representation !== "native-scan" + || source.sampling !== "deterministic-even-index" + ) { + throw new DatasetGatewayContractError("Dataset preview contract несовместим"); + } + const sourcePointCount = number(source.source_point_count, "source_point_count"); + const pointCount = number(source.point_count, "point_count"); + const pointsXyzM = array(source.points_xyz_m, "points_xyz_m").map( + (value, index): [number, number, number] => { + const point = array(value, `points_xyz_m[${index}]`); + if ( + point.length !== 3 + || point.some((coordinate) => + typeof coordinate !== "number" || !Number.isFinite(coordinate) + ) + ) { + throw new DatasetGatewayContractError( + `points_xyz_m[${index}]: некорректная точка`, + ); + } + return [point[0] as number, point[1] as number, point[2] as number]; + }, + ); + const remission0To255 = integers( + source.remission_0_to_255, + "remission_0_to_255", + 255, + ); + const semanticLabelIds = integers( + source.semantic_label_ids, + "semantic_label_ids", + 65_535, + ); + const semanticRgb0To255 = integers( + source.semantic_rgb_0_to_255, + "semantic_rgb_0_to_255", + 255, + ); + const groundTruthGround = integers( + source.ground_truth_ground, + "ground_truth_ground", + 1, + ); + if ( + pointCount < 1 + || pointCount > 50_000 + || pointCount > sourcePointCount + || pointsXyzM.length !== pointCount + || remission0To255.length !== pointCount + || semanticLabelIds.length !== pointCount + || semanticRgb0To255.length !== pointCount * 3 + || groundTruthGround.length !== pointCount + ) { + throw new DatasetGatewayContractError("Dataset preview arrays не выровнены"); + } + const classes = array(source.classes, "classes").map((value, index) => { + const item = record(value, `classes[${index}]`); + return { + labelId: number(item.label_id, "class.label_id"), + className: string(item.class_name, "class.class_name", true), + hex: string(item.hex, "class.hex"), + challengeCategoryId: number( + item.challenge_category_id, + "class.challenge_category_id", + ), + challengeCategoryName: string( + item.challenge_category_name, + "class.challenge_category_name", + true, + ), + }; + }); + const safety = record(source.safety, "safety"); + if ( + safety.visualization_only !== true + || safety.navigation_or_safety_accepted !== false + ) { + throw new DatasetGatewayContractError("Dataset preview safety boundary нарушен"); + } + return { + sourceId: "goose-3d/v2025-08-22", + frameId: string(source.frame_id, "frame_id", true), + sourcePointCount, + pointCount, + pointsXyzM, + remission0To255, + semanticLabelIds, + semanticRgb0To255, + groundTruthGround, + classes, + }; +} + +export function parseDatasetGroundComparison( + value: unknown, +): DatasetGroundComparison { + const source = record(value, "Ground comparison"); + if ( + source.schema_version !== "missioncore.dataset-ground-comparison-preview/v1" + || source.source_id !== "goose-3d/v2025-08-22" + || source.sampling !== "deterministic-even-index" + ) { + throw new DatasetGatewayContractError("Ground comparison contract несовместим"); + } + const pointCount = number(source.point_count, "point_count"); + const currentGround = integers(source.current_ground, "current_ground", 1); + const groundTruthGround = integers( + source.ground_truth_ground, + "ground_truth_ground", + 1, + ); + const evaluated = integers(source.evaluated, "evaluated", 1); + const disagreement = integers(source.disagreement, "disagreement", 1); + if ( + pointCount < 1 + || pointCount > 50_000 + || currentGround.length !== pointCount + || groundTruthGround.length !== pointCount + || evaluated.length !== pointCount + || disagreement.length !== pointCount + ) { + throw new DatasetGatewayContractError("Ground comparison arrays не выровнены"); + } + const metrics = record(source.metrics, "metrics"); + const fraction = (key: string): number => { + const value = number(metrics[key], `metrics.${key}`); + if (value > 1) { + throw new DatasetGatewayContractError(`metrics.${key}: ожидалась доля`); + } + return value; + }; + const provider = record(source.provider, "provider"); + if ( + provider.provider_id !== "missioncore-local-percentile-ground/v1" + || provider.ground_truth !== false + || !/^[a-f0-9]{64}$/.test( + string(provider.implementation_sha256, "provider.implementation_sha256"), + ) + ) { + throw new DatasetGatewayContractError("Ground comparison provider несовместим"); + } + const safety = record(source.safety, "safety"); + if ( + safety.qualification_only !== true + || safety.navigation_or_safety_accepted !== false + ) { + throw new DatasetGatewayContractError("Ground comparison safety boundary нарушен"); + } + return { + sourceId: "goose-3d/v2025-08-22", + frameId: string(source.frame_id, "frame_id", true), + pointCount, + currentGround, + groundTruthGround, + evaluated, + disagreement, + metrics: { + precision: fraction("precision"), + recall: fraction("recall"), + f1: fraction("f1"), + groundIou: fraction("ground_iou"), + accuracy: fraction("accuracy"), + artificialGroundRecall: fraction("artificial_ground_recall"), + naturalGroundRecall: fraction("natural_ground_recall"), + obstacleNonGroundRecall: fraction("obstacle_non_ground_recall"), + }, + latencyMs: number(source.latency_ms, "latency_ms"), + }; +} + async function responseJson(response: Response): Promise { if (!response.ok) { throw new Error(`Dataset Gateway HTTP ${response.status}`); @@ -241,3 +590,30 @@ export async function fetchDatasetGatewayCatalog( }); return parseDatasetGatewayCatalog(await responseJson(response)); } + +export async function fetchDatasetNativeScanPreview( + options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {}, +): Promise { + const fetcher = options.fetcher ?? fetch; + const response = await fetcher("/api/v1/lidar/dataset-gateway/preview", { + method: "GET", + headers: { Accept: "application/json" }, + signal: options.signal, + }); + return parseDatasetNativeScanPreview(await responseJson(response)); +} + +export async function fetchDatasetGroundComparison( + options: { signal?: AbortSignal; fetcher?: DatasetFetch } = {}, +): Promise { + const fetcher = options.fetcher ?? fetch; + const response = await fetcher( + "/api/v1/lidar/dataset-gateway/ground-comparison", + { + method: "GET", + headers: { Accept: "application/json" }, + signal: options.signal, + }, + ); + return parseDatasetGroundComparison(await responseJson(response)); +} diff --git a/apps/control-station/src/styles/responsive.css b/apps/control-station/src/styles/responsive.css index 21b17b0..8bf5cf6 100644 --- a/apps/control-station/src/styles/responsive.css +++ b/apps/control-station/src/styles/responsive.css @@ -122,6 +122,15 @@ grid-template-columns: 1fr; } + .dataset-preview > header { + align-items: flex-start; + flex-direction: column; + } + + .dataset-preview__metrics { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .lidar-device-context__verdict { border-top: 1px solid var(--station-hairline); border-left: 0; diff --git a/apps/control-station/src/styles/workspaces.css b/apps/control-station/src/styles/workspaces.css index a14a27c..a5f632b 100644 --- a/apps/control-station/src/styles/workspaces.css +++ b/apps/control-station/src/styles/workspaces.css @@ -1098,6 +1098,128 @@ gap: 0.5rem; } +.dataset-download { + position: relative; + grid-column: 1 / -1; + overflow: hidden; + height: 1.7rem; + border-radius: 0.65rem; + background: rgb(255 255 255 / 0.045); +} + +.dataset-download > span { + position: absolute; + inset: 0 auto 0 0; + background: rgb(255 255 255 / 0.16); + transition: width 300ms ease; +} + +.dataset-download p { + position: relative; + z-index: 1; + display: grid; + height: 100%; + place-items: center; + color: var(--nodedc-text-secondary); + font-size: 0.58rem; +} + +.dataset-preview { + display: grid; + gap: 0.8rem; + padding: 1rem; + border-radius: 0.9rem; + background: var(--station-panel); +} + +.dataset-preview > header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 1rem; +} + +.dataset-preview h2 { + margin-top: 0.22rem; + color: var(--nodedc-text-primary); + font-size: 0.92rem; +} + +.dataset-preview .lidar-ground-scene { + min-height: 31rem; + border: 0; +} + +.dataset-preview__metrics { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + overflow: hidden; + border-radius: 0.7rem; + background: rgb(255 255 255 / 0.025); +} + +.dataset-preview__metrics div { + display: grid; + gap: 0.2rem; + padding: 0.65rem 0.75rem; +} + +.dataset-preview__metrics div + div { + border-left: 1px solid var(--station-hairline); +} + +.dataset-preview__metrics dt { + color: var(--nodedc-text-muted); + font-size: 0.52rem; +} + +.dataset-preview__metrics dd { + margin: 0; + color: var(--nodedc-text-primary); + font-size: 0.72rem; + font-weight: 600; +} + +.dataset-preview__comparison-note { + color: var(--nodedc-text-muted); + font-size: 0.6rem; +} + +.dataset-preview__legend { + display: flex; + overflow-x: auto; + gap: 0.65rem; + padding: 0.1rem 0.15rem 0.15rem; + scrollbar-width: thin; +} + +.dataset-preview__legend span { + display: inline-flex; + flex: 0 0 auto; + align-items: center; + gap: 0.32rem; + color: var(--nodedc-text-muted); + font-size: 0.55rem; +} + +.dataset-preview__legend i { + width: 0.48rem; + height: 0.48rem; + border-radius: 50%; +} + +.dataset-preview__legend .dataset-legend-ground { + background: rgb(199 230 107); +} + +.dataset-preview__legend .dataset-legend-other { + background: rgb(74 84 97); +} + +.dataset-preview__legend .dataset-legend-error { + background: rgb(255 79 56); +} + .dataset-contract { overflow: hidden; background: var(--station-panel); diff --git a/apps/control-station/src/workspaces/DatasetGatewayWorkspace.tsx b/apps/control-station/src/workspaces/DatasetGatewayWorkspace.tsx index a9f9c61..21a7124 100644 --- a/apps/control-station/src/workspaces/DatasetGatewayWorkspace.tsx +++ b/apps/control-station/src/workspaces/DatasetGatewayWorkspace.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { Button, GlassSurface, @@ -7,8 +7,16 @@ import { import { fetchDatasetGatewayCatalog, + fetchDatasetGroundComparison, + fetchDatasetNativeScanPreview, type DatasetGatewayCatalog, + type DatasetGroundComparison, + type DatasetNativeScanPreview, } from "../core/lidar/datasetGateway"; +import { + LidarGroundPointCloud, + type LidarGroundViewMode, +} from "./LidarGroundPointCloud"; const representationLabels: Record = { "native-scan": "Исходный скан", @@ -22,11 +30,39 @@ function errorMessage(error: unknown): string { : "Каталог датасетов недоступен."; } +function admissionLabel(status: DatasetGatewayCatalog["source"]["admissionStatus"]) { + const labels: Record = { + "blocked-storage-policy": "Worker не настроен", + "ready-for-download": "Готов к загрузке", + downloading: "Загружается", + downloaded: "Загружен", + verifying: "Проверяется", + verified: "Проверен", + "frame-ready": "Первый кадр готов", + }; + return labels[status]; +} + +function formatBytes(value: number): string { + return new Intl.NumberFormat("ru-RU", { + style: "unit", + unit: "gigabyte", + maximumFractionDigits: 2, + }).format(value / 1_000_000_000); +} + export function DatasetGatewayWorkspace() { const [catalog, setCatalog] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [reloadGeneration, setReloadGeneration] = useState(0); + const [previewOpen, setPreviewOpen] = useState(false); + const [preview, setPreview] = useState(null); + const [comparison, setComparison] = useState(null); + const [previewError, setPreviewError] = useState(null); + const [comparisonError, setComparisonError] = useState(null); + const [previewMode, setPreviewMode] = + useState("semantic"); useEffect(() => { const controller = new AbortController(); @@ -47,6 +83,73 @@ export function DatasetGatewayWorkspace() { return () => controller.abort(); }, [reloadGeneration]); + useEffect(() => { + if (catalog?.source.admissionStatus !== "downloading") return; + const timer = window.setTimeout( + () => setReloadGeneration((value) => value + 1), + 5_000, + ); + return () => window.clearTimeout(timer); + }, [catalog]); + + useEffect(() => { + if (!previewOpen || preview) return; + const controller = new AbortController(); + setPreviewError(null); + void fetchDatasetNativeScanPreview({ signal: controller.signal }) + .then((value) => { + if (!controller.signal.aborted) setPreview(value); + }) + .catch((loadError: unknown) => { + if (!controller.signal.aborted) setPreviewError(errorMessage(loadError)); + }); + return () => controller.abort(); + }, [previewOpen, preview]); + + useEffect(() => { + if (!previewOpen || comparison) return; + const controller = new AbortController(); + setComparisonError(null); + void fetchDatasetGroundComparison({ signal: controller.signal }) + .then((value) => { + if (!controller.signal.aborted) setComparison(value); + }) + .catch((loadError: unknown) => { + if (!controller.signal.aborted) { + setComparisonError(errorMessage(loadError)); + } + }); + return () => controller.abort(); + }, [comparison, previewOpen]); + + const previewFrame = useMemo(() => { + if (!preview) return null; + const emptyMask = new Array(preview.pointCount).fill(0); + const comparisonAligned = ( + comparison + && comparison.frameId === preview.frameId + && comparison.pointCount === preview.pointCount + ) ? comparison : null; + return { + pointCount: preview.pointCount, + pointsXyzM: preview.pointsXyzM, + intensity0To255: preview.remission0To255, + semanticRgb0To255: preview.semanticRgb0To255, + masks: { + currentGround: comparisonAligned?.currentGround ?? emptyMask, + currentAssigned: comparisonAligned?.evaluated ?? emptyMask, + candidateGround: comparisonAligned?.groundTruthGround ?? emptyMask, + candidateAssigned: comparisonAligned?.evaluated ?? emptyMask, + disagreement: comparisonAligned?.disagreement ?? emptyMask, + groundTruthGround: preview.groundTruthGround, + }, + }; + }, [comparison, preview]); + + const downloadProgress = catalog?.source.archive + ? catalog.source.archive.bytesTransferred / catalog.source.archive.totalBytes + : null; + return (
@@ -117,35 +220,56 @@ export function DatasetGatewayWorkspace() {
Состояние
-
Не загружен
+
{admissionLabel(catalog.source.admissionStatus)}
Validation
{catalog.source.validationArchiveGb} ГБ
-
Разметка
-
{catalog.source.superclasses.length} классов
+
Группы
+
{catalog.source.superclasses.length} superclass
Лицензия
{catalog.source.license}
+ {catalog.source.admissionStatus === "downloading" + && catalog.source.archive + && downloadProgress !== null ? ( +
+ +

+ {formatBytes(catalog.source.archive.bytesTransferred)} + {" из "} + {formatBytes(catalog.source.archive.totalBytes)} +

+
+ ) : null}

- {catalog.storage.admitted - ? "Следующий шаг: загрузить validation archive на worker и проверить hash/license." - : "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."} + {catalog.source.admissionStatus === "frame-ready" + ? "Native scan и разметка прошли admission. Можно открыть независимый ground truth." + : catalog.source.admissionStatus === "downloading" + ? "Архив остаётся на D worker. После загрузки проверим ZIP, лицензию, digest и point-label alignment." + : catalog.storage.admitted + ? "Следующий шаг: загрузить validation archive на worker и проверить hash/license." + : "Сначала нужно допустить Dataset Root на диске D worker. Сейчас открывать нечего."}

+ ))} +
+ + {comparison ? ( +
+
+
Ground IoU
+
{(comparison.metrics.groundIou * 100).toFixed(1)}%
+
+
+
Precision
+
{(comparison.metrics.precision * 100).toFixed(1)}%
+
+
+
Recall
+
{(comparison.metrics.recall * 100).toFixed(1)}%
+
+
+
Latency
+
{comparison.latencyMs.toFixed(0)} ms
+
+
+ ) : comparisonError ? ( +

+ Current baseline пока не рассчитан: {comparisonError} +

+ ) : null} + {previewFrame ? ( + <> + +
+ {previewMode === "semantic" ? ( + preview?.classes.map((item) => ( + + + {item.className} + + )) + ) : previewMode === "ground-truth" ? ( + <> + ground truth + остальные точки + + ) : previewMode === "current" ? ( + <> + current ground + current non-ground + + ) : previewMode === "disagreement" ? ( + <> + ошибка относительно labels + совпадение + + ) : ( + Remission нормализован только для визуального просмотра 0–255 + )} +
+ + ) : ( +
+ + {previewError ? "Preview недоступен" : "Читаем frame"} + +

{previewError ?? "Проверяем point alignment и разметку."}

+
+ )} +
+ ) : null} +
Технический контракт diff --git a/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx b/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx index 073104e..3899efa 100644 --- a/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx +++ b/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx @@ -6,18 +6,22 @@ export type LidarGroundViewMode = | "intensity" | "current" | "candidate" - | "disagreement"; + | "disagreement" + | "semantic" + | "ground-truth"; export interface LidarGroundPointCloudFrame { pointCount: number; pointsXyzM: Array<[number, number, number]>; intensity0To255: number[] | null; + semanticRgb0To255?: number[] | null; masks: { currentGround: number[]; currentAssigned: number[]; candidateGround: number[]; candidateAssigned: number[]; disagreement: number[]; + groundTruthGround?: number[]; }; } @@ -68,6 +72,24 @@ function frameColors( neutral, neutral, ); + } else if (mode === "semantic") { + const semantic = frame.semanticRgb0To255; + setRgb( + colors, + offset, + (semantic?.[offset] ?? 96) / 255, + (semantic?.[offset + 1] ?? 96) / 255, + (semantic?.[offset + 2] ?? 96) / 255, + ); + } else if (mode === "ground-truth") { + const groundTruth = frame.masks.groundTruthGround?.[index] === 1; + setRgb( + colors, + offset, + groundTruth ? 0.78 : 0.29, + groundTruth ? 0.9 : 0.33, + groundTruth ? 0.42 : 0.38, + ); } else if (mode === "current") { setRgb( colors, @@ -88,14 +110,17 @@ function frameColors( candidate ? 1 : 0.43, ); } - } else if (current && candidate) { - setRgb(colors, offset, 0.73, 1, 0.29); - } else if (current) { - setRgb(colors, offset, 1, 0.64, 0.18); - } else if (candidate) { - setRgb(colors, offset, 0.24, 0.84, 1); + } else if (mode === "disagreement") { + const disagreement = frame.masks.disagreement[index] === 1; + setRgb( + colors, + offset, + disagreement ? 1 : 0.24, + disagreement ? 0.31 : 0.29, + disagreement ? 0.22 : 0.35, + ); } else { - setRgb(colors, offset, 0.24, 0.29, 0.35); + setRgb(colors, offset, candidate ? 0.24 : 0.29, candidate ? 0.84 : 0.36, 0.43); } } return colors; diff --git a/apps/control-station/test/datasetGateway.test.mjs b/apps/control-station/test/datasetGateway.test.mjs index dead629..09ecc06 100644 --- a/apps/control-station/test/datasetGateway.test.mjs +++ b/apps/control-station/test/datasetGateway.test.mjs @@ -5,7 +5,11 @@ import { createServer } from "vite"; let server; let parseDatasetGatewayCatalog; +let parseDatasetNativeScanPreview; +let parseDatasetGroundComparison; let fetchDatasetGatewayCatalog; +let fetchDatasetNativeScanPreview; +let fetchDatasetGroundComparison; let DatasetGatewayContractError; before(async () => { @@ -16,7 +20,11 @@ before(async () => { }); ({ parseDatasetGatewayCatalog, + parseDatasetNativeScanPreview, + parseDatasetGroundComparison, fetchDatasetGatewayCatalog, + fetchDatasetNativeScanPreview, + fetchDatasetGroundComparison, DatasetGatewayContractError, } = await server.ssrLoadModule("/src/core/lidar/datasetGateway.ts")); }); @@ -27,7 +35,7 @@ after(async () => { function catalog(overrides = {}) { return { - schema_version: "missioncore.dataset-gateway-catalog/v1", + schema_version: "missioncore.dataset-gateway-catalog/v2", access: "read-only", storage: { configured: false, @@ -35,6 +43,8 @@ function catalog(overrides = {}) { required_wsl_root: "/mnt/d/NDC_MISSIONCORE/datasets", admitted: false, status: "blocked-storage-policy", + attestation: "none", + manifest_valid: false, path_exposed: false, }, sources: [{ @@ -54,6 +64,8 @@ function catalog(overrides = {}) { }, admission: { status: "blocked-storage-policy", + archive: null, + frame: null, }, }], representations: [ @@ -137,3 +149,99 @@ test("fetches the read-only gateway endpoint", async () => { assert.equal(calls[0].init.method, "GET"); assert.equal(parsed.source.displayName, "GOOSE 3D"); }); + +test("decodes one bounded point-aligned native-scan preview", async () => { + const payload = { + schema_version: "missioncore.dataset-native-scan-preview/v1", + source_id: "goose-3d/v2025-08-22", + frame_id: "frame-1", + representation: "native-scan", + sampling: "deterministic-even-index", + source_point_count: 2, + point_count: 2, + points_xyz_m: [[1, 2, 3], [4, 5, 6]], + remission_0_to_255: [0, 255], + semantic_label_ids: [23, 38], + semantic_rgb_0_to_255: [255, 47, 128, 1, 51, 73], + ground_truth_ground: [1, 0], + classes: [ + { + label_id: 23, + class_name: "asphalt", + hex: "#ff2f80", + challenge_category_id: 2, + challenge_category_name: "artificial_ground", + }, + ], + safety: { + visualization_only: true, + navigation_or_safety_accepted: false, + }, + }; + const parsed = parseDatasetNativeScanPreview(payload); + assert.equal(parsed.pointCount, 2); + assert.deepEqual(parsed.groundTruthGround, [1, 0]); + + const fetched = await fetchDatasetNativeScanPreview({ + fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }), + }); + assert.equal(fetched.frameId, "frame-1"); + + payload.semantic_rgb_0_to_255.pop(); + assert.throws( + () => parseDatasetNativeScanPreview(payload), + DatasetGatewayContractError, + ); +}); + +test("decodes a point-aligned current-vs-ground-truth comparison", async () => { + const payload = { + schema_version: "missioncore.dataset-ground-comparison-preview/v1", + source_id: "goose-3d/v2025-08-22", + frame_id: "frame-1", + sampling: "deterministic-even-index", + point_count: 2, + current_ground: [1, 1], + ground_truth_ground: [1, 0], + evaluated: [1, 1], + disagreement: [0, 1], + metrics: { + true_positive: 1, + false_positive: 1, + false_negative: 0, + true_negative: 0, + precision: 0.5, + recall: 1, + f1: 2 / 3, + ground_iou: 0.5, + accuracy: 0.5, + artificial_ground_recall: 1, + natural_ground_recall: 0, + obstacle_non_ground_recall: 1, + }, + latency_ms: 12.5, + provider: { + provider_id: "missioncore-local-percentile-ground/v1", + implementation_sha256: "a".repeat(64), + ground_truth: false, + }, + safety: { + qualification_only: true, + navigation_or_safety_accepted: false, + }, + }; + const parsed = parseDatasetGroundComparison(payload); + assert.equal(parsed.metrics.groundIou, 0.5); + assert.deepEqual(parsed.disagreement, [0, 1]); + + const fetched = await fetchDatasetGroundComparison({ + fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }), + }); + assert.equal(fetched.latencyMs, 12.5); + + payload.disagreement.pop(); + assert.throws( + () => parseDatasetGroundComparison(payload), + DatasetGatewayContractError, + ); +}); diff --git a/docs/14_LIDAR_DATASET_GATEWAY.md b/docs/14_LIDAR_DATASET_GATEWAY.md index dbbcf25..73ccbbe 100644 --- a/docs/14_LIDAR_DATASET_GATEWAY.md +++ b/docs/14_LIDAR_DATASET_GATEWAY.md @@ -15,12 +15,16 @@ This prevents tuning an algorithm until a visually dense vendor map merely looks plausible. It also keeps work reusable across Gazebo, Unreal, public datasets and future real onboard sensors. -## Current S0 slice +## Current admitted slice Implemented now: -- `missioncore.dataset-gateway-catalog/v1`, exposed read-only at +- `missioncore.dataset-gateway-catalog/v2`, exposed read-only at `GET /api/v1/lidar/dataset-gateway`; +- path-free `missioncore.dataset-admission/v1` worker evidence instead of a + static React status; +- a bounded `missioncore.dataset-native-scan-preview/v1`, exposed read-only at + `GET /api/v1/lidar/dataset-gateway/preview`; - explicit `native-scan`, `normalized-scan` and `rolling-local-map` representations; - a lossless GOOSE/SemanticKITTI frame reader for little-endian float32 XYZI @@ -31,12 +35,16 @@ Implemented now: - worker storage admission for `D:\NDC_MISSIONCORE\datasets` and `/mnt/d/NDC_MISSIONCORE/datasets`; - a dedicated, honest dataset catalog in **Полигон → Датасеты**; +- live admission states (`downloading`, `verifying`, `frame-ready`) and a real + `Открыть` action; +- one admitted GOOSE validation frame with source colors, normalized remission + and an independent ground-truth view; - a separate **Парк → Диагностика LiDAR** surface containing only real sensor recordings and their operational evidence. -Not implemented in S0: +Not implemented: -- no automatic 3.3 GB validation archive download; +- no browser-triggered large-artifact download; - no implicit coordinate conversion; - no fake ring/timestamp reconstruction for K1 MQTT evidence; - no model training or production promotion; @@ -55,9 +63,11 @@ The gateway is not part of device quality diagnostics. - **Полигон → Прогоны** owns the resulting algorithm comparison, metrics, provenance and decision. -Before real dataset bytes exist, the catalog shows `Не загружен`, explains the -blocked storage gate and keeps `Открыть` / `Создать прогон` disabled. An -architecture contract must not look like an executable dataset tool. +Before real dataset bytes exist, the catalog explains the blocked storage gate +and keeps `Открыть` / `Создать прогон` disabled. `Открыть` becomes available +only after the worker manifest says `frame-ready` and the bounded preview passes +its own point-alignment and safety checks. `Создать прогон` remains disabled +until an executable comparison profile exists. ## Why public recordings look different @@ -65,6 +75,11 @@ GOOSE stores one VLS-128 revolution per annotated `.bin` file. A rotating multi-channel sensor produces discrete scan lines, so a single sensor-frame view looks like sparse rings. +The admitted `goose_3d_val.zip` uses `lidar/val/.../*_vls128.bin` even though +some GOOSE documentation examples call the point-cloud root `velodyne`. Mission +Core accepts either declared root but still requires an exactly aligned +`labels/val/.../*_goose.label`. + The current field review is explicitly an accumulated map-frame window. It combines many source publications after pose registration. This fills surfaces and hides the original scan pattern. The external K1 stream is also already a @@ -130,14 +145,16 @@ time; TTL/dynamic filtering prevents stale ghosts. ## First dataset sequence -1. Configure `MISSIONCORE_DATASET_ROOT=/mnt/d/NDC_MISSIONCORE/datasets` on the - Windows/WSL worker. -2. Verify free space and record archive size/hash/license. -3. Download only the GOOSE 3D validation archive first (published size 3.3 GB). -4. Import one labeled frame and expose it in React as `native-scan`. -5. Show native remission and ground-truth superclass coloring. +1. [Done] Admit `/mnt/d/NDC_MISSIONCORE/datasets` on the Windows/WSL worker. +2. [Done] Verify free space and record archive size/hash/license. +3. [Done] Download only the GOOSE 3D validation archive first (published size + 3.3 GB). +4. [Done] Import one labeled frame and expose it in React as `native-scan`. +5. [Done] Show native remission, the original semantic palette and + ground-truth superclass coloring. 6. Add a declared GOOSE frame/mounting profile and produce `normalized-scan`. -7. Run current ground heuristic and Patchwork++ against independent labels. +7. [Current baseline done; Patchwork++ blocked on mounting evidence] Run the + current ground heuristic and Patchwork++ against independent labels. 8. Add sensor-degradation profiles for range, FOV, density, noise and dropout. 9. Only after the one-frame contract passes, expand to the validation split and add a rolling-map sequence with localization evidence. @@ -152,10 +169,70 @@ time; TTL/dynamic filtering prevents stale ghosts. - [x] React exposes an honest catalog/empty state before dataset bytes exist. - [x] Dataset Gateway is isolated from real-sensor diagnostics and placed under Polygon qualification. -- [ ] Worker D root configured. -- [ ] GOOSE validation archive hash recorded. -- [ ] First real labeled frame visible in React. +- [x] Worker D root configured. +- [x] GOOSE validation archive hash recorded. +- [x] First real labeled frame visible in React. - [ ] Coordinate and mounting profile admitted. +- [x] Current local-percentile baseline measured against ground truth. - [ ] Patchwork++ accuracy measured against ground truth. - [ ] Sensor-degradation matrix qualified. - [ ] Rolling local map with pose/TTL/dynamic policy qualified. + +## First GOOSE admission evidence + +- worker root: canonical D-only root, path never exposed by the HTTP API; +- source URL: + `https://goose-dataset.de/storage/goose_3d_val.zip`; +- observed archive bytes: `3,498,402,435`; +- observed SHA-256: + `0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f`; +- archive integrity: ZIP structure, bounded expansion, safe paths, one LICENSE, + one label mapping and aligned point/label frames; +- license artifact: Creative Commons Attribution-ShareAlike 4.0 International; +- first deterministic frame: + `2022-07-22_flight__0071_1658494234334310308`; +- source points: `169,883`; +- semantic classes present: `17`; +- independently labeled artificial/natural ground: `45,968` points + (`27.0586%`); +- browser preview: deterministic even-index sample of `50,000` points, + visualization-only. + +GOOSE does not publish a checksum beside this download. The recorded SHA-256 is +therefore Mission Core's observed content pin after a TLS download, not a claim +of vendor-signed authenticity. Any future byte change creates a new admission +decision rather than silently replacing this installation. + +## First independent ground result + +The current `missioncore-local-percentile-ground/v1` provider was run against +all `169,883` points, excluding `353` GOOSE void points from scoring: + +- Ground IoU: `49.6165%`; +- precision: `73.5892%`; +- recall: `60.3659%`; +- F1: `66.3249%`; +- artificial-ground recall: `95.2364%`; +- natural-ground recall: `49.4516%`; +- obstacle non-ground recall: `79.6145%`; +- provider latency on the worker: `3,858.16 ms`. + +This is the first concrete product value from the public dataset. The current +heuristic looks plausible on a dense map and is strong on artificial ground, +but it misses about half of the independently labeled natural ground and is far +from a real-time full-scan provider. The UI exposes `Current` and `Ошибки` +views, so the failure geometry is inspectable instead of being hidden behind +one aggregate score. + +The provider now uses a bounded grid-neighborhood index while preserving the +previous exact-radius result. This removes the previous all-points scan for +every occupied cell, but the measured latency still classifies it as a +diagnostic baseline rather than an onboard candidate. + +Patchwork++ is intentionally not scored yet. The validation ZIP contains XYZI, +labels, mapping, LICENSE and CHANGELOG but no numeric TF/mounting calibration. +GOOSE documents the VLS-128 as a roof LiDAR and publishes a separate MuCAR-3 TF +tree, but the graph image alone is not physical-height evidence. The next gate +is to admit the numeric transform from `base_link_ground` to +`sensor/lidar/vls128_roof`, declare the source axis convention, and only then +run Patchwork++. diff --git a/docs/adr/0021-dataset-gateway-representation-boundary.md b/docs/adr/0021-dataset-gateway-representation-boundary.md index 5e90fd6..0529a72 100644 --- a/docs/adr/0021-dataset-gateway-representation-boundary.md +++ b/docs/adr/0021-dataset-gateway-representation-boundary.md @@ -49,6 +49,12 @@ The gateway: - admits storage only under `D:\NDC_MISSIONCORE\datasets` or its WSL mirror; - never promotes K1 `lio_pcl` to `native-scan`. +Large-artifact execution remains worker-local. The browser receives only a +path-free admission manifest and a bounded visualization preview. The canonical +archive, extracted source frame, labels, mapping and LICENSE remain under the +worker D root. SSH/SCP may mirror the small manifest and preview during the +recorded laboratory bootstrap; it is not the product data plane. + The normalized pipeline is: ```text @@ -73,8 +79,9 @@ unavailable rather than inventing timestamps. into device evidence. - Dataset catalog, preview and qualification-run entry live under Polygon; Data remains the source-of-record and artifact store. -- Until bytes are installed, the UI exposes an explicit empty state rather - than presenting this architecture contract as an interactive tool. +- Until bytes are installed, the UI exposes an explicit empty state. After a + `frame-ready` worker admission, the same surface becomes an interactive + source/remission/ground-truth viewer without changing the representation. - Dataset and device inputs can share downstream algorithms only after their normalized contracts match. - Sensor adaptation may change range, FOV, point density, noise and dropout for @@ -85,6 +92,13 @@ unavailable rather than inventing timestamps. independent labels. - Stable operator visualization is owned by rolling-map policy, not by the ground classifier. +- Public labels may qualify an algorithm independently of the production + sensor. The first GOOSE frame exposed a `49.62%` Ground IoU and only `49.45%` + natural-ground recall for the current local-percentile baseline; these are + diagnostic results, not production promotion. +- A dataset label contract does not imply a mounting contract. Patchwork++ + remains blocked until the numeric GOOSE roof-LiDAR transform and physical + height are admitted from source evidence. ## Primary references diff --git a/pyproject.toml b/pyproject.toml index e3055e1..10d5e05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ missioncore-plugin-sdk = { path = "packages/plugin-sdk", editable = true } [project.scripts] k1link = "k1link.device_plugins.xgrids_k1.cli:app" +missioncore-datasets = "k1link.datasets.cli:app" missioncore-sim = "k1link.simulation.cli:app" [dependency-groups] diff --git a/src/k1link/compute/lidar_ground.py b/src/k1link/compute/lidar_ground.py index 9dbcd0e..9d16e55 100644 --- a/src/k1link/compute/lidar_ground.py +++ b/src/k1link/compute/lidar_ground.py @@ -10,15 +10,25 @@ import re import shutil import time from collections.abc import Mapping -from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from types import ModuleType -from typing import Any, Final, Protocol +from typing import Any, Final import numpy as np import numpy.typing as npt +from k1link.ground_segmentation import ( + DEFAULT_GROUND_BENCHMARK_PROFILE, + GroundBenchmarkProfile, + GroundSegmentation, + GroundSegmenter, + LocalPercentileGroundSegmenter, +) +from k1link.ground_segmentation import ( + GroundSegmentationError as LidarGroundError, +) + from .lidar_contract import LidarContractError, sensor_frame_xyzi from .lidar_replay import LidarReplayPackV2 @@ -41,189 +51,6 @@ _ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$ _SHA256 = re.compile(r"^[a-f0-9]{64}$") _GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$") -BoolArray = npt.NDArray[np.bool_] -FloatArray = npt.NDArray[np.floating[Any]] - - -class LidarGroundError(ValueError): - """Ground benchmark evidence violates the diagnostic-only contract.""" - - -@dataclass(frozen=True, slots=True) -class GroundBenchmarkProfile: - profile_id: str = "k1-vendor-map-ground-ab/v1" - pose_binding_threshold_ms: float = 100.0 - current_cell_size_m: float = 0.5 - current_local_radius_m: float = 2.5 - current_lower_percentile: float = 8.0 - current_maximum_below_ground_m: float = 0.25 - current_maximum_above_ground_m: float = 0.12 - current_minimum_local_points: int = 8 - patchwork_sensor_height_proxy_m: float = 0.0 - patchwork_map_vertical_origin_offset_m: float = 0.0 - patchwork_height_evidence: str = "missing" - patchwork_minimum_range_m: float = 0.1 - patchwork_maximum_range_m: float = 20.0 - - def __post_init__(self) -> None: - finite_values = ( - self.pose_binding_threshold_ms, - self.current_cell_size_m, - self.current_local_radius_m, - self.current_lower_percentile, - self.current_maximum_below_ground_m, - self.current_maximum_above_ground_m, - self.patchwork_sensor_height_proxy_m, - self.patchwork_map_vertical_origin_offset_m, - self.patchwork_minimum_range_m, - self.patchwork_maximum_range_m, - ) - if not all(math.isfinite(value) for value in finite_values): - raise LidarGroundError("Ground benchmark profile must be finite") - if ( - not 0 < self.pose_binding_threshold_ms <= 10_000 - or not 0 < self.current_cell_size_m <= 100 - or not 0 < self.current_local_radius_m <= 1_000 - or not 0 <= self.current_lower_percentile <= 100 - or not 0 <= self.current_maximum_below_ground_m <= 100 - or not 0 <= self.current_maximum_above_ground_m <= 100 - or not 1 <= self.current_minimum_local_points <= 1_000_000 - or not 0 <= self.patchwork_sensor_height_proxy_m <= 10 - or not -10 <= self.patchwork_map_vertical_origin_offset_m <= 10 - or not 0 <= self.patchwork_minimum_range_m < self.patchwork_maximum_range_m <= 1_000 - or self.patchwork_height_evidence - not in {"missing", "operator-estimated", "runtime-calibrated"} - ): - raise LidarGroundError("Ground benchmark profile is invalid") - if self.patchwork_height_evidence == "missing" and ( - self.patchwork_sensor_height_proxy_m != 0 - or self.patchwork_map_vertical_origin_offset_m != 0 - ): - raise LidarGroundError("Ground benchmark cannot apply height without height evidence") - if ( - self.patchwork_height_evidence != "missing" - and self.patchwork_sensor_height_proxy_m <= 0 - ): - raise LidarGroundError( - "Ground benchmark height evidence requires a positive sensor height" - ) - - def to_dict(self) -> dict[str, object]: - return { - "schema_version": "missioncore.lidar-ground-benchmark-profile/v1", - "profile_id": self.profile_id, - "pose_binding": { - "basis": "nearest-recorded-host-monotonic-arrival", - "threshold_ms": self.pose_binding_threshold_ms, - }, - "current_baseline": { - "provider_id": "missioncore-local-percentile-ground/v1", - "derived_from": "local-ground-relative-object-support-v1", - "cell_size_m": self.current_cell_size_m, - "local_radius_m": self.current_local_radius_m, - "lower_percentile": self.current_lower_percentile, - "maximum_below_ground_m": self.current_maximum_below_ground_m, - "maximum_above_ground_m": self.current_maximum_above_ground_m, - "minimum_local_points": self.current_minimum_local_points, - }, - "candidate": { - "provider_id": "patchworkpp/v1.4.1", - "sensor_height_proxy_m": self.patchwork_sensor_height_proxy_m, - "map_vertical_origin_offset_m": (self.patchwork_map_vertical_origin_offset_m), - "height_evidence": self.patchwork_height_evidence, - "minimum_range_m": self.patchwork_minimum_range_m, - "maximum_range_m": self.patchwork_maximum_range_m, - "enable_rnr": True, - "enable_rvpf": True, - "enable_tgr": True, - }, - "input_normalization": { - "current": "vendor-map-xyz", - "candidate": "best-effort-map-to-lidar-pose-inversion", - "physical_sensor_height_known": ( - self.patchwork_height_evidence == "runtime-calibrated" - ), - "sensor_scan_geometry_known": False, - }, - "authority": { - "commands_enabled": False, - "navigation_or_safety_accepted": False, - }, - } - - -DEFAULT_GROUND_BENCHMARK_PROFILE: Final = GroundBenchmarkProfile() - - -@dataclass(frozen=True, slots=True) -class GroundSegmentation: - ground_mask: BoolArray - assigned_mask: BoolArray - latency_ms: float - - -class GroundSegmenter(Protocol): - @property - def identity(self) -> Mapping[str, object]: ... - - def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ... - - -class LocalPercentileGroundSegmenter: - """Full-frame diagnostic extension of the existing E19 local ground heuristic.""" - - def __init__(self, profile: GroundBenchmarkProfile) -> None: - self.profile = profile - - @property - def identity(self) -> Mapping[str, object]: - return { - "provider_id": "missioncore-local-percentile-ground/v1", - "implementation_sha256": _sha256(Path(__file__).resolve(strict=True)), - "ground_truth": False, - } - - def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: - points = _xyzi(xyzi) - started = time.perf_counter_ns() - xyz = points[:, :3].astype(np.float64, copy=False) - cell_size = self.profile.current_cell_size_m - cell_keys = np.floor(xyz[:, :2] / cell_size).astype(np.int64) - unique_cells, inverse = np.unique(cell_keys, axis=0, return_inverse=True) - ground = np.zeros(points.shape[0], dtype=np.bool_) - global_ground_z = float(np.percentile(xyz[:, 2], self.profile.current_lower_percentile)) - radius_squared = self.profile.current_local_radius_m**2 - for cell_index in range(unique_cells.shape[0]): - point_indices = np.flatnonzero(inverse == cell_index) - if point_indices.size == 0: - continue - center_xy = np.median(xyz[point_indices, :2], axis=0) - delta_xy = xyz[:, :2] - center_xy - local = xyz[ - np.einsum("ij,ij->i", delta_xy, delta_xy) <= radius_squared, - 2, - ] - ground_z = ( - float( - np.percentile( - local, - self.profile.current_lower_percentile, - ) - ) - if local.size >= self.profile.current_minimum_local_points - else global_ground_z - ) - z = xyz[point_indices, 2] - ground[point_indices] = ( - z >= ground_z - self.profile.current_maximum_below_ground_m - ) & (z <= ground_z + self.profile.current_maximum_above_ground_m) - latency_ms = (time.perf_counter_ns() - started) / 1_000_000 - return GroundSegmentation( - ground_mask=ground, - assigned_mask=np.ones(points.shape[0], dtype=np.bool_), - latency_ms=latency_ms, - ) - class PatchworkPPGroundSegmenter: """Runtime-only adapter for the pinned official Patchwork++ Python binding.""" diff --git a/src/k1link/datasets/__init__.py b/src/k1link/datasets/__init__.py index 0d25e08..4506af9 100644 --- a/src/k1link/datasets/__init__.py +++ b/src/k1link/datasets/__init__.py @@ -2,16 +2,38 @@ from k1link.datasets.gateway import ( DATASET_GATEWAY_CATALOG_SCHEMA, + DatasetAdmissionError, DatasetFrameError, DatasetPointFrame, + configured_dataset_admission_manifest, + configured_dataset_ground_preview, + configured_dataset_preview, dataset_gateway_catalog, + read_dataset_admission_manifest, + read_dataset_ground_preview, + read_dataset_native_scan_preview, read_semantic_kitti_frame, ) +from k1link.datasets.goose_benchmark import ( + GOOSE_GROUND_BENCHMARK_SCHEMA, + GOOSE_GROUND_PREVIEW_SCHEMA, + benchmark_goose_current_ground, +) __all__ = [ "DATASET_GATEWAY_CATALOG_SCHEMA", + "DatasetAdmissionError", "DatasetFrameError", "DatasetPointFrame", + "GOOSE_GROUND_BENCHMARK_SCHEMA", + "GOOSE_GROUND_PREVIEW_SCHEMA", + "benchmark_goose_current_ground", + "configured_dataset_admission_manifest", + "configured_dataset_ground_preview", + "configured_dataset_preview", "dataset_gateway_catalog", + "read_dataset_admission_manifest", + "read_dataset_ground_preview", + "read_dataset_native_scan_preview", "read_semantic_kitti_frame", ] diff --git a/src/k1link/datasets/cli.py b/src/k1link/datasets/cli.py new file mode 100644 index 0000000..c75f138 --- /dev/null +++ b/src/k1link/datasets/cli.py @@ -0,0 +1,67 @@ +"""Operator CLI for worker-local public dataset admission.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer + +from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation +from k1link.datasets.goose_benchmark import benchmark_goose_current_ground + +app = typer.Typer( + add_completion=False, + help="Admit public perception datasets without moving source data off worker D.", +) + + +@app.command("admit-goose-validation") +def admit_goose_validation_command( + dataset_root: Annotated[ + Path, + typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True), + ], + archive: Annotated[ + Path | None, + typer.Option("--archive", exists=True, dir_okay=False, resolve_path=True), + ] = None, + preview_points: Annotated[ + int, + typer.Option("--preview-points", min=1, max=50_000), + ] = 50_000, +) -> None: + """Verify the pinned GOOSE validation archive and import one native scan.""" + + try: + manifest = admit_goose_validation( + dataset_root, + archive_path=archive, + preview_points=preview_points, + ) + except GooseAdmissionError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True)) + + +@app.command("benchmark-goose-current-ground") +def benchmark_goose_current_ground_command( + dataset_root: Annotated[ + Path, + typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True), + ], +) -> None: + """Score the existing Mission Core ground baseline against GOOSE labels.""" + + try: + report = benchmark_goose_current_ground(dataset_root) + except GooseAdmissionError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True)) + + +if __name__ == "__main__": + app() diff --git a/src/k1link/datasets/gateway.py b/src/k1link/datasets/gateway.py index 2823b1f..a237920 100644 --- a/src/k1link/datasets/gateway.py +++ b/src/k1link/datasets/gateway.py @@ -7,16 +7,25 @@ map construction are separate, provenance-bearing products. from __future__ import annotations +import json import os from dataclasses import dataclass from pathlib import Path, PureWindowsPath -from typing import Final, Literal +from typing import Any, Final, Literal import numpy as np import numpy.typing as npt -DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v1" +DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2" +DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1" +DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1" +DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1" DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT" +DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST" +DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW" +DATASET_GROUND_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_GROUND_PREVIEW" +MAX_STATE_BYTES: Final = 64 * 1024 +MAX_PREVIEW_BYTES: Final = 16 * 1024**2 Representation = Literal["native-scan", "normalized-scan", "rolling-local-map"] @@ -24,6 +33,10 @@ class DatasetFrameError(ValueError): """A dataset frame cannot satisfy its declared lossless source contract.""" +class DatasetAdmissionError(ValueError): + """A worker admission artifact violates the path-free dataset contract.""" + + @dataclass(frozen=True) class DatasetPointFrame: """One SemanticKITTI-compatible labeled LiDAR frame. @@ -91,6 +104,21 @@ def _configured_dataset_root() -> Path | None: return Path(raw).expanduser().absolute() if raw else None +def configured_dataset_admission_manifest() -> Path | None: + raw = os.environ.get(DATASET_ADMISSION_MANIFEST_ENV, "").strip() + return Path(raw).expanduser().absolute() if raw else None + + +def configured_dataset_preview() -> Path | None: + raw = os.environ.get(DATASET_PREVIEW_ENV, "").strip() + return Path(raw).expanduser().absolute() if raw else None + + +def configured_dataset_ground_preview() -> Path | None: + raw = os.environ.get(DATASET_GROUND_PREVIEW_ENV, "").strip() + return Path(raw).expanduser().absolute() if raw else None + + def _is_worker_d_storage(root: Path | None) -> bool: if root is None: return False @@ -104,20 +132,61 @@ def _is_worker_d_storage(root: Path | None) -> bool: ) -def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, object]: +def dataset_gateway_catalog( + dataset_root: Path | None = None, + admission_manifest_path: Path | None = None, +) -> dict[str, object]: """Return the path-free, read-only ingress plan and current admission state.""" root = dataset_root if dataset_root is not None else _configured_dataset_root() - storage_admitted = _is_worker_d_storage(root) + manifest_path = ( + admission_manifest_path + if admission_manifest_path is not None + else configured_dataset_admission_manifest() + ) + admission: dict[str, Any] | None = None + admission_error: str | None = None + if manifest_path is not None: + try: + admission = read_dataset_admission_manifest(manifest_path) + except DatasetAdmissionError: + admission_error = "worker-manifest-invalid" + locally_admitted = _is_worker_d_storage(root) + worker_admitted = bool(admission and admission["storage"]["admitted"]) + storage_admitted = locally_admitted or worker_admitted + source_status = ( + str(admission["status"]) + if admission is not None + else "ready-for-download" + if storage_admitted + else "blocked-storage-policy" + ) + next_action = ( + str(admission["next_action"]) + if admission is not None + else "repair-worker-admission-manifest" + if admission_error is not None + else "download-goose-validation-to-d" + if storage_admitted + else "configure-dataset-root-on-worker-d" + ) return { "schema_version": DATASET_GATEWAY_CATALOG_SCHEMA, "access": "read-only", "storage": { - "configured": root is not None, + "configured": root is not None or manifest_path is not None, "required_windows_root": r"D:\NDC_MISSIONCORE\datasets", "required_wsl_root": "/mnt/d/NDC_MISSIONCORE/datasets", "admitted": storage_admitted, "status": "ready" if storage_admitted else "blocked-storage-policy", + "attestation": ( + "worker-manifest" + if worker_admitted + else "local-worker-path" + if locally_admitted + else "none" + ), + "manifest_valid": admission is not None, "path_exposed": False, }, "sources": [ @@ -149,12 +218,12 @@ def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, objec "test_archive_gb": 3.3, }, "admission": { - "status": ( - "ready-for-download" if storage_admitted else "blocked-storage-policy" - ), + "status": source_status, "native_scan": "ready-after-download", "normalized_scan": "requires-explicit-frame-and-mounting-contract", "rolling_local_map": "requires-pose-timing-and-map-policy", + "archive": admission["archive"] if admission is not None else None, + "frame": admission["frame"] if admission is not None else None, }, } ], @@ -219,9 +288,316 @@ def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, objec "reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan", } ], - "next_action": ( - "download-goose-validation-to-d" - if storage_admitted - else "configure-dataset-root-on-worker-d" - ), + "next_action": next_action, } + + +def read_dataset_admission_manifest(path: Path) -> dict[str, Any]: + document = _bounded_json_object(path, MAX_STATE_BYTES, "dataset admission manifest") + if ( + document.get("schema_version") != DATASET_ADMISSION_SCHEMA + or document.get("source_id") != "goose-3d/v2025-08-22" + or set(document) + != { + "schema_version", + "source_id", + "observed_at_utc", + "status", + "storage", + "archive", + "license", + "frame", + "next_action", + } + ): + raise DatasetAdmissionError("dataset admission manifest identity is incompatible") + status = document["status"] + if status not in {"downloading", "downloaded", "verifying", "verified", "frame-ready"}: + raise DatasetAdmissionError("dataset admission status is incompatible") + storage = _object(document["storage"], "storage") + if ( + set(storage) + != {"policy", "admitted", "canonical_root", "path_exposed"} + or storage.get("policy") != "worker-d-only" + or storage.get("admitted") is not True + or storage.get("canonical_root") is not True + or storage.get("path_exposed") is not False + ): + raise DatasetAdmissionError("dataset storage admission is incompatible") + archive = _object(document["archive"], "archive") + if ( + set(archive) + != { + "filename", + "source_url", + "bytes_transferred", + "total_bytes", + "size_bytes", + "sha256", + "integrity", + "vendor_checksum_available", + } + or archive.get("filename") != "goose_3d_val.zip" + or archive.get("source_url") + != "https://goose-dataset.de/storage/goose_3d_val.zip" + or archive.get("vendor_checksum_available") is not False + ): + raise DatasetAdmissionError("dataset archive admission is incompatible") + transferred = _nonnegative_integer(archive["bytes_transferred"], "bytes_transferred") + total = _positive_integer(archive["total_bytes"], "total_bytes") + if transferred > total: + raise DatasetAdmissionError("dataset download progress is incompatible") + size = archive["size_bytes"] + digest = archive["sha256"] + if size is not None: + _positive_integer(size, "size_bytes") + if digest is not None and ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise DatasetAdmissionError("dataset archive digest is incompatible") + license_value = _object(document["license"], "license") + if ( + set(license_value) != {"spdx", "artifact_present"} + or license_value.get("spdx") != "CC-BY-SA-4.0" + or not isinstance(license_value.get("artifact_present"), bool) + ): + raise DatasetAdmissionError("dataset license admission is incompatible") + frame = document["frame"] + if status == "frame-ready": + _validate_frame_admission(_object(frame, "frame")) + elif frame is not None: + raise DatasetAdmissionError("dataset frame must be absent before frame-ready") + for key in ("observed_at_utc", "next_action"): + if not isinstance(document[key], str) or not document[key]: + raise DatasetAdmissionError(f"{key} is incompatible") + return document + + +def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]: + document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview") + if ( + document.get("schema_version") != DATASET_PREVIEW_SCHEMA + or document.get("source_id") != "goose-3d/v2025-08-22" + or document.get("representation") != "native-scan" + or document.get("sampling") != "deterministic-even-index" + ): + raise DatasetAdmissionError("dataset preview identity is incompatible") + point_count = _positive_integer(document.get("point_count"), "point_count") + source_point_count = _positive_integer( + document.get("source_point_count"), "source_point_count" + ) + if point_count > 50_000 or point_count > source_point_count: + raise DatasetAdmissionError("dataset preview point count is incompatible") + points = document.get("points_xyz_m") + remission = document.get("remission_0_to_255") + semantic_ids = document.get("semantic_label_ids") + semantic_rgb = document.get("semantic_rgb_0_to_255") + ground = document.get("ground_truth_ground") + if ( + not isinstance(points, list) + or len(points) != point_count + or not isinstance(remission, list) + or len(remission) != point_count + or not isinstance(semantic_ids, list) + or len(semantic_ids) != point_count + or not isinstance(semantic_rgb, list) + or len(semantic_rgb) != point_count * 3 + or not isinstance(ground, list) + or len(ground) != point_count + ): + raise DatasetAdmissionError("dataset preview arrays are not point-aligned") + for point in points: + if ( + not isinstance(point, list) + or len(point) != 3 + or any( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + for value in point + ) + ): + raise DatasetAdmissionError("dataset preview point is incompatible") + for values, maximum, label in ( + (remission, 255, "remission"), + (semantic_ids, 65_535, "semantic label"), + (semantic_rgb, 255, "semantic color"), + (ground, 1, "ground mask"), + ): + if any( + not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum + for value in values + ): + raise DatasetAdmissionError(f"dataset preview {label} is incompatible") + classes = document.get("classes") + if not isinstance(classes, list) or len(classes) > 64: + raise DatasetAdmissionError("dataset preview class catalog is incompatible") + safety = _object(document.get("safety"), "safety") + if safety != { + "visualization_only": True, + "navigation_or_safety_accepted": False, + }: + raise DatasetAdmissionError("dataset preview safety boundary is incompatible") + if not isinstance(document.get("frame_id"), str) or not document["frame_id"]: + raise DatasetAdmissionError("dataset preview frame id is incompatible") + return document + + +def read_dataset_ground_preview(path: Path) -> dict[str, Any]: + document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview") + if ( + document.get("schema_version") != DATASET_GROUND_PREVIEW_SCHEMA + or document.get("source_id") != "goose-3d/v2025-08-22" + or document.get("sampling") != "deterministic-even-index" + ): + raise DatasetAdmissionError("dataset ground preview identity is incompatible") + point_count = _positive_integer(document.get("point_count"), "point_count") + if point_count > 50_000: + raise DatasetAdmissionError("dataset ground preview point count is incompatible") + for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"): + values = document.get(key) + if ( + not isinstance(values, list) + or len(values) != point_count + or any( + not isinstance(value, int) + or isinstance(value, bool) + or value not in (0, 1) + for value in values + ) + ): + raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible") + metrics = _object(document.get("metrics"), "metrics") + expected_metrics = { + "true_positive", + "false_positive", + "false_negative", + "true_negative", + "precision", + "recall", + "f1", + "ground_iou", + "accuracy", + "artificial_ground_recall", + "natural_ground_recall", + "obstacle_non_ground_recall", + } + if set(metrics) != expected_metrics: + raise DatasetAdmissionError("dataset ground metrics are incompatible") + for key, value in metrics.items(): + if key in {"true_positive", "false_positive", "false_negative", "true_negative"}: + _nonnegative_integer(value, f"metrics.{key}") + elif ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + or not 0 <= value <= 1 + ): + raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible") + latency = document.get("latency_ms") + if ( + not isinstance(latency, (int, float)) + or isinstance(latency, bool) + or not np.isfinite(latency) + or latency < 0 + ): + raise DatasetAdmissionError("dataset ground latency is incompatible") + provider = _object(document.get("provider"), "provider") + if ( + set(provider) != {"provider_id", "implementation_sha256", "ground_truth"} + or provider.get("provider_id") != "missioncore-local-percentile-ground/v1" + or provider.get("ground_truth") is not False + ): + raise DatasetAdmissionError("dataset ground provider is incompatible") + digest = provider.get("implementation_sha256") + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise DatasetAdmissionError("dataset ground provider digest is incompatible") + safety = _object(document.get("safety"), "safety") + if safety != { + "qualification_only": True, + "navigation_or_safety_accepted": False, + }: + raise DatasetAdmissionError("dataset ground safety boundary is incompatible") + if not isinstance(document.get("frame_id"), str) or not document["frame_id"]: + raise DatasetAdmissionError("dataset ground frame id is incompatible") + return document + + +def _validate_frame_admission(frame: dict[str, Any]) -> None: + if ( + set(frame) + != { + "frame_id", + "representation", + "point_count", + "semantic_class_count", + "ground_truth_ground_points", + "ground_truth_ground_fraction", + "preview_point_count", + "preview_sha256", + "preview_available", + } + or frame.get("representation") != "native-scan" + or frame.get("preview_available") is not True + ): + raise DatasetAdmissionError("dataset frame admission is incompatible") + point_count = _positive_integer(frame["point_count"], "frame.point_count") + ground_count = _nonnegative_integer( + frame["ground_truth_ground_points"], "frame.ground_truth_ground_points" + ) + if ground_count > point_count: + raise DatasetAdmissionError("dataset ground-truth count is incompatible") + for key in ("semantic_class_count", "preview_point_count"): + value = _positive_integer(frame[key], key) + if value > point_count: + raise DatasetAdmissionError(f"{key} is incompatible") + fraction = frame["ground_truth_ground_fraction"] + if ( + not isinstance(fraction, (int, float)) + or isinstance(fraction, bool) + or not 0 <= fraction <= 1 + ): + raise DatasetAdmissionError("dataset ground-truth fraction is incompatible") + digest = frame["preview_sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise DatasetAdmissionError("dataset preview digest is incompatible") + if not isinstance(frame["frame_id"], str) or not frame["frame_id"]: + raise DatasetAdmissionError("dataset frame id is incompatible") + + +def _bounded_json_object(path: Path, maximum_bytes: int, label: str) -> dict[str, Any]: + try: + if not path.is_file() or not 0 < path.stat().st_size <= maximum_bytes: + raise DatasetAdmissionError(f"{label} size is incompatible") + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DatasetAdmissionError(f"{label} cannot be decoded") from exc + return _object(document, label) + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise DatasetAdmissionError(f"{label} must be an object") + return value + + +def _positive_integer(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + raise DatasetAdmissionError(f"{label} must be a positive integer") + return value + + +def _nonnegative_integer(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise DatasetAdmissionError(f"{label} must be a non-negative integer") + return value diff --git a/src/k1link/datasets/goose_admission.py b/src/k1link/datasets/goose_admission.py new file mode 100644 index 0000000..288bc34 --- /dev/null +++ b/src/k1link/datasets/goose_admission.py @@ -0,0 +1,519 @@ +"""Fail-closed admission of one GOOSE 3D validation frame. + +The large source archive and extracted native frame remain on the worker D +drive. The only portable products are a path-free admission manifest and a +bounded visualization preview; neither is an alternative copy of the dataset. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import os +import shutil +import stat +import tempfile +import zipfile +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any, Final + +import numpy as np + +from k1link.datasets.gateway import DatasetFrameError, read_semantic_kitti_frame + +GOOSE_SOURCE_ID: Final = "goose-3d/v2025-08-22" +GOOSE_VALIDATION_URL: Final = "https://goose-dataset.de/storage/goose_3d_val.zip" +GOOSE_LICENSE: Final = "CC-BY-SA-4.0" +GOOSE_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1" +GOOSE_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1" +GOOSE_ARCHIVE_FILENAME: Final = "goose_3d_val.zip" +GOOSE_ARCHIVE_OBSERVED_BYTES: Final = 3_498_402_435 +MAX_ARCHIVE_ENTRIES: Final = 100_000 +MAX_UNCOMPRESSED_BYTES: Final = 128 * 1024**3 +MAX_METADATA_BYTES: Final = 4 * 1024**2 +MAX_PREVIEW_POINTS: Final = 50_000 +GOOSE_CHALLENGE_GROUPS: Final[dict[str, frozenset[str]]] = { + "other": frozenset({"undefined", "ego_vehicle", "outlier"}), + "artificial_structures": frozenset({"building", "wall", "bridge", "tunnel"}), + "artificial_ground": frozenset( + { + "cobble", + "bikeway", + "pedestrian_crossing", + "road_marking", + "sidewalk", + "curb", + "asphalt", + } + ), + "natural_ground": frozenset( + {"snow", "leaves", "gravel", "soil", "low_grass", "water"} + ), + "obstacle": frozenset( + { + "traffic_cone", + "obstacle", + "street_light", + "road_block", + "traffic_light", + "boom_barrier", + "rail_track", + "debris", + "animal", + "rock", + "fence", + "guard_rail", + "pole", + "traffic_sign", + "misc_sign", + "barrier_tape", + "wire", + "container", + "barrel", + "pipe", + } + ), + "vehicle": frozenset( + { + "car", + "bicycle", + "bus", + "motorcycle", + "truck", + "on_rails", + "caravan", + "trailer", + "kick_scooter", + "heavy_machinery", + "military_vehicle", + } + ), + "vegetation": frozenset( + { + "forest", + "bush", + "moss", + "tree_crown", + "tree_trunk", + "crops", + "high_grass", + "scenery_vegetation", + "hedge", + "tree_root", + } + ), + "human": frozenset({"person", "rider"}), + "sky": frozenset({"sky"}), +} +GOOSE_CHALLENGE_IDS: Final = { + name: identifier + for identifier, name in enumerate( + ( + "other", + "artificial_structures", + "artificial_ground", + "natural_ground", + "obstacle", + "vehicle", + "vegetation", + "human", + "sky", + ) + ) +} + + +class GooseAdmissionError(RuntimeError): + """The source archive cannot satisfy the pinned GOOSE admission contract.""" + + +def admit_goose_validation( + dataset_root: Path, + *, + archive_path: Path | None = None, + preview_points: int = MAX_PREVIEW_POINTS, +) -> dict[str, Any]: + """Verify a GOOSE validation archive and admit one deterministic frame.""" + + root = dataset_root.expanduser().absolute() + if not _is_canonical_worker_root(root): + raise GooseAdmissionError("GOOSE admission requires the canonical worker D dataset root") + archive = ( + archive_path.expanduser().absolute() + if archive_path is not None + else root / "goose-3d/v2025-08-22/archives" / GOOSE_ARCHIVE_FILENAME + ) + if not archive.is_file(): + raise GooseAdmissionError("GOOSE validation archive is unavailable") + size_bytes = archive.stat().st_size + if size_bytes != GOOSE_ARCHIVE_OBSERVED_BYTES: + raise GooseAdmissionError( + "GOOSE validation archive size differs from the pinned observed release" + ) + if not 1 <= preview_points <= MAX_PREVIEW_POINTS: + raise GooseAdmissionError("preview point limit is outside the admitted range") + + archive_sha256 = _sha256_file(archive) + install_root = root / "goose-3d/v2025-08-22/installs" / archive_sha256 + state_root = root / "state" + state_root.mkdir(parents=True, exist_ok=True) + + try: + with zipfile.ZipFile(archive) as source: + members = _admitted_members(source) + point_member, label_member = _first_frame_pair(members) + mapping_member = _unique_metadata_member(members, "goose_label_mapping.csv") + license_member = _unique_metadata_member(members, "LICENSE") + mapping_bytes = _bounded_member_bytes(source, mapping_member) + license_bytes = _bounded_member_bytes(source, license_member) + labels = parse_goose_label_mapping(mapping_bytes) + license_text = license_bytes.decode("utf-8", errors="replace") + if "Attribution-ShareAlike 4.0 International" not in license_text: + raise GooseAdmissionError("archive LICENSE does not declare CC BY-SA 4.0") + + frame_id = _frame_id(point_member.filename) + frame_root = install_root / "frames" / frame_id + frame_root.mkdir(parents=True, exist_ok=True) + point_path = frame_root / "points.bin" + label_path = frame_root / "labels.label" + mapping_path = install_root / "goose_label_mapping.csv" + license_path = install_root / "LICENSE" + _extract_once(source, point_member, point_path) + _extract_once(source, label_member, label_path) + _write_once(mapping_path, mapping_bytes) + _write_once(license_path, license_bytes) + except (OSError, zipfile.BadZipFile) as exc: + raise GooseAdmissionError("GOOSE archive could not be verified") from exc + + try: + frame = read_semantic_kitti_frame(point_path, label_path) + except DatasetFrameError as exc: + raise GooseAdmissionError("first GOOSE frame violates XYZI/label alignment") from exc + unknown_labels = sorted( + int(value) for value in np.unique(frame.semantic_labels) if int(value) not in labels + ) + if unknown_labels: + raise GooseAdmissionError("frame contains semantic ids absent from the label mapping") + + preview = _native_scan_preview( + frame_id, + frame.points_xyz_m, + frame.remission, + frame.semantic_labels, + labels, + maximum_points=preview_points, + ) + preview_path = install_root / "previews" / f"{frame_id}.json" + _atomic_json(preview_path, preview) + preview_sha256 = _sha256_file(preview_path) + present_classes = Counter(int(value) for value in frame.semantic_labels) + ground_points = sum( + count + for label_id, count in present_classes.items() + if labels[label_id]["challenge_category_id"] in (2, 3) + ) + manifest = { + "schema_version": GOOSE_ADMISSION_SCHEMA, + "source_id": GOOSE_SOURCE_ID, + "observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "status": "frame-ready", + "storage": { + "policy": "worker-d-only", + "admitted": True, + "canonical_root": True, + "path_exposed": False, + }, + "archive": { + "filename": GOOSE_ARCHIVE_FILENAME, + "source_url": GOOSE_VALIDATION_URL, + "bytes_transferred": size_bytes, + "total_bytes": size_bytes, + "size_bytes": size_bytes, + "sha256": archive_sha256, + "integrity": "zip-structure-and-content-digest", + "vendor_checksum_available": False, + }, + "license": { + "spdx": GOOSE_LICENSE, + "artifact_present": True, + }, + "frame": { + "frame_id": frame_id, + "representation": "native-scan", + "point_count": frame.point_count, + "semantic_class_count": len(present_classes), + "ground_truth_ground_points": ground_points, + "ground_truth_ground_fraction": ground_points / frame.point_count, + "preview_point_count": int(preview["point_count"]), + "preview_sha256": preview_sha256, + "preview_available": True, + }, + "next_action": "review-first-native-scan", + } + _atomic_json(state_root / "goose-3d-v2025-08-22.json", manifest) + return manifest + + +def _is_canonical_worker_root(root: Path) -> bool: + normalized = str(root).replace("\\", "/").rstrip("/").lower() + return normalized == "/mnt/d/ndc_missioncore/datasets" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024**2), b""): + digest.update(chunk) + except OSError as exc: + raise GooseAdmissionError("dataset artifact cannot be hashed") from exc + return digest.hexdigest() + + +def _admitted_members(source: zipfile.ZipFile) -> tuple[zipfile.ZipInfo, ...]: + members = tuple(source.infolist()) + if not members or len(members) > MAX_ARCHIVE_ENTRIES: + raise GooseAdmissionError("archive entry count is outside the admitted range") + total_bytes = 0 + for member in members: + path = PurePosixPath(member.filename.replace("\\", "/")) + if path.is_absolute() or ".." in path.parts or not path.parts: + raise GooseAdmissionError("archive contains an unsafe member path") + mode = member.external_attr >> 16 + if stat.S_ISLNK(mode): + raise GooseAdmissionError("archive contains a symbolic link") + total_bytes += member.file_size + if total_bytes > MAX_UNCOMPRESSED_BYTES: + raise GooseAdmissionError("archive expands beyond the admitted size") + return members + + +def _first_frame_pair( + members: tuple[zipfile.ZipInfo, ...], +) -> tuple[zipfile.ZipInfo, zipfile.ZipInfo]: + points = { + _frame_id(member.filename): member + for member in members + if not member.is_dir() + and any( + prefix in "/" + member.filename.replace("\\", "/") + for prefix in ("/lidar/val/", "/velodyne/val/") + ) + and member.filename.endswith("_vls128.bin") + } + labels = { + _frame_id(member.filename): member + for member in members + if not member.is_dir() + and "/labels/val/" in "/" + member.filename.replace("\\", "/") + and member.filename.endswith("_goose.label") + } + shared = sorted(points.keys() & labels.keys()) + if not shared: + raise GooseAdmissionError("archive has no aligned GOOSE validation frame") + frame_id = shared[0] + return points[frame_id], labels[frame_id] + + +def _frame_id(filename: str) -> str: + name = PurePosixPath(filename.replace("\\", "/")).name + for suffix in ("_vls128.bin", "_goose.label"): + if name.endswith(suffix): + identifier = name[: -len(suffix)] + if identifier: + return identifier + raise GooseAdmissionError("GOOSE frame filename is incompatible") + + +def _unique_metadata_member( + members: tuple[zipfile.ZipInfo, ...], + filename: str, +) -> zipfile.ZipInfo: + matches = [member for member in members if PurePosixPath(member.filename).name == filename] + if len(matches) != 1 or matches[0].is_dir(): + raise GooseAdmissionError(f"archive must contain exactly one {filename}") + return matches[0] + + +def _bounded_member_bytes(source: zipfile.ZipFile, member: zipfile.ZipInfo) -> bytes: + if member.file_size <= 0 or member.file_size > MAX_METADATA_BYTES: + raise GooseAdmissionError("archive metadata size is outside the admitted range") + value = source.read(member) + if len(value) != member.file_size: + raise GooseAdmissionError("archive metadata was truncated") + return value + + +def _extract_once(source: zipfile.ZipFile, member: zipfile.ZipInfo, target: Path) -> None: + if target.exists(): + if target.is_file() and target.stat().st_size == member.file_size: + return + raise GooseAdmissionError("immutable extracted artifact already exists with another size") + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + with source.open(member) as member_source: + shutil.copyfileobj(member_source, temporary, length=1024**2) + temporary.flush() + os.fsync(temporary.fileno()) + except Exception: + temporary_path.unlink(missing_ok=True) + raise + if temporary_path.stat().st_size != member.file_size: + temporary_path.unlink(missing_ok=True) + raise GooseAdmissionError("extracted artifact size does not match the archive") + os.replace(temporary_path, target) + + +def _write_once(target: Path, value: bytes) -> None: + if target.exists(): + if target.is_file() and target.read_bytes() == value: + return + raise GooseAdmissionError("immutable metadata artifact already exists with other content") + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + temporary.write(value) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, target) + + +def read_goose_label_mapping(path: Path) -> dict[int, dict[str, Any]]: + try: + value = path.read_bytes() + except OSError as exc: + raise GooseAdmissionError("GOOSE label mapping is unavailable") from exc + if not 0 < len(value) <= MAX_METADATA_BYTES: + raise GooseAdmissionError("GOOSE label mapping size is outside the admitted range") + return parse_goose_label_mapping(value) + + +def parse_goose_label_mapping(value: bytes) -> dict[int, dict[str, Any]]: + try: + text = value.decode("utf-8-sig") + rows = tuple(csv.DictReader(text.splitlines())) + except (UnicodeDecodeError, csv.Error) as exc: + raise GooseAdmissionError("GOOSE label mapping cannot be decoded") from exc + required = { + "class_name", + "label_key", + "hex", + } + if not rows or not required.issubset(rows[0]): + raise GooseAdmissionError("GOOSE label mapping columns are incompatible") + labels: dict[int, dict[str, Any]] = {} + try: + for row in rows: + label_id = int(row["label_key"]) + color = row["hex"].strip().lower() + if label_id in labels or not 0 <= label_id <= 65_535: + raise GooseAdmissionError("GOOSE label ids are invalid") + if len(color) != 7 or color[0] != "#": + raise GooseAdmissionError("GOOSE label color is invalid") + int(color[1:], 16) + labels[label_id] = { + "class_name": row["class_name"].strip(), + "hex": color, + **_challenge_category(row["class_name"].strip()), + } + except (KeyError, TypeError, ValueError) as exc: + raise GooseAdmissionError("GOOSE label mapping rows are invalid") from exc + return labels + + +def _challenge_category(class_name: str) -> dict[str, Any]: + matches = [ + name for name, members in GOOSE_CHALLENGE_GROUPS.items() if class_name in members + ] + if len(matches) != 1: + raise GooseAdmissionError("GOOSE class is absent from the pinned challenge mapping") + name = matches[0] + return { + "challenge_category_id": GOOSE_CHALLENGE_IDS[name], + "challenge_category_name": name, + } + + +def _native_scan_preview( + frame_id: str, + points_xyz_m: np.ndarray[Any, Any], + remission: np.ndarray[Any, Any], + semantic_labels: np.ndarray[Any, Any], + labels: dict[int, dict[str, Any]], + *, + maximum_points: int, +) -> dict[str, Any]: + point_count = int(points_xyz_m.shape[0]) + sample_count = min(point_count, maximum_points) + indices = np.linspace(0, point_count - 1, sample_count, dtype=np.int64) + sampled_points = points_xyz_m[indices] + sampled_remission = remission[indices] + sampled_labels = semantic_labels[indices] + remission_min = float(np.min(sampled_remission)) + remission_max = float(np.max(sampled_remission)) + remission_span = remission_max - remission_min + if not math.isfinite(remission_span): + raise GooseAdmissionError("GOOSE remission range is non-finite") + if remission_span > 0: + scaled_remission = np.rint( + (sampled_remission - remission_min) * (255.0 / remission_span) + ).astype(np.uint8) + else: + scaled_remission = np.zeros(sample_count, dtype=np.uint8) + semantic_rgb: list[int] = [] + ground_mask: list[int] = [] + for value in sampled_labels: + label = labels[int(value)] + color = label["hex"] + semantic_rgb.extend( + (int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16)) + ) + ground_mask.append(1 if label["challenge_category_id"] in (2, 3) else 0) + present_ids = sorted({int(value) for value in sampled_labels}) + return { + "schema_version": GOOSE_PREVIEW_SCHEMA, + "source_id": GOOSE_SOURCE_ID, + "frame_id": frame_id, + "representation": "native-scan", + "sampling": "deterministic-even-index", + "source_point_count": point_count, + "point_count": sample_count, + "points_xyz_m": sampled_points.tolist(), + "remission_0_to_255": scaled_remission.tolist(), + "semantic_label_ids": sampled_labels.astype(np.uint16).tolist(), + "semantic_rgb_0_to_255": semantic_rgb, + "ground_truth_ground": ground_mask, + "classes": [ + { + "label_id": label_id, + **labels[label_id], + } + for label_id in present_ids + ], + "safety": { + "visualization_only": True, + "navigation_or_safety_accepted": False, + }, + } + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + encoded = ( + json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + + b"\n" + ) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + temporary.write(encoded) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, path) diff --git a/src/k1link/datasets/goose_benchmark.py b/src/k1link/datasets/goose_benchmark.py new file mode 100644 index 0000000..99ebf64 --- /dev/null +++ b/src/k1link/datasets/goose_benchmark.py @@ -0,0 +1,270 @@ +"""Independent-label ground benchmark for an admitted GOOSE native scan.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import time +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.datasets.gateway import read_dataset_admission_manifest, read_semantic_kitti_frame +from k1link.datasets.goose_admission import ( + GOOSE_SOURCE_ID, + GooseAdmissionError, + read_goose_label_mapping, +) +from k1link.ground_segmentation import ( + DEFAULT_GROUND_BENCHMARK_PROFILE, + GroundBenchmarkProfile, + LocalPercentileGroundSegmenter, +) + +GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1" +GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1" +MAX_PREVIEW_POINTS: Final = 50_000 + + +def benchmark_goose_current_ground( + dataset_root: Path, + *, + profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, + preview_points: int = MAX_PREVIEW_POINTS, +) -> dict[str, Any]: + """Score the existing Mission Core ground baseline on one admitted frame.""" + + root = dataset_root.expanduser().absolute() + if not _is_worker_dataset_root(root): + raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root") + manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json") + if manifest["status"] != "frame-ready": + raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking") + archive_sha256 = str(manifest["archive"]["sha256"]) + frame_id = str(manifest["frame"]["frame_id"]) + install = root / "goose-3d/v2025-08-22/installs" / archive_sha256 + frame_root = install / "frames" / frame_id + frame = read_semantic_kitti_frame( + frame_root / "points.bin", + frame_root / "labels.label", + ) + labels = read_goose_label_mapping(install / "goose_label_mapping.csv") + categories = np.asarray( + [labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels], + dtype=np.uint8, + ) + evaluated = categories != 0 + ground_truth = (categories == 2) | (categories == 3) + xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype( + np.float32, + copy=False, + ) + segmenter = LocalPercentileGroundSegmenter(profile) + started = time.perf_counter_ns() + prediction = segmenter.segment(xyzi) + wall_latency_ms = (time.perf_counter_ns() - started) / 1_000_000 + metrics = _binary_metrics( + prediction.ground_mask, + ground_truth, + evaluated, + ) + metrics.update( + { + "artificial_ground_recall": _recall( + prediction.ground_mask, + categories == 2, + ), + "natural_ground_recall": _recall( + prediction.ground_mask, + categories == 3, + ), + "obstacle_non_ground_recall": _recall( + ~prediction.ground_mask, + categories == 4, + ), + } + ) + profile_document = profile.to_dict() + identity_document = { + "source_id": GOOSE_SOURCE_ID, + "archive_sha256": archive_sha256, + "frame_id": frame_id, + "source_point_count": frame.point_count, + "profile": profile_document, + "provider": dict(segmenter.identity), + } + identity_sha256 = _canonical_sha256(identity_document) + output_root = install / "benchmarks" / f"current-ground-{identity_sha256}" + output_root.mkdir(parents=True, exist_ok=True) + report_path = output_root / "report.json" + if report_path.is_file(): + try: + existing = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise GooseAdmissionError("existing GOOSE benchmark report is invalid") from exc + if ( + not isinstance(existing, dict) + or existing.get("schema_version") != GOOSE_GROUND_BENCHMARK_SCHEMA + or existing.get("identity_sha256") != identity_sha256 + or not (output_root / "prediction.npz").is_file() + or not (output_root / "preview.json").is_file() + ): + raise GooseAdmissionError("existing GOOSE benchmark is incomplete") + return existing + prediction_path = output_root / "prediction.npz" + _atomic_npz( + prediction_path, + current_ground=prediction.ground_mask.astype(np.uint8), + ground_truth_ground=ground_truth.astype(np.uint8), + evaluated=evaluated.astype(np.uint8), + ) + prediction_sha256 = _sha256_file(prediction_path) + + sample_count = min(frame.point_count, preview_points) + indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64) + preview = { + "schema_version": GOOSE_GROUND_PREVIEW_SCHEMA, + "source_id": GOOSE_SOURCE_ID, + "frame_id": frame_id, + "sampling": "deterministic-even-index", + "point_count": sample_count, + "current_ground": prediction.ground_mask[indices].astype(np.uint8).tolist(), + "ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(), + "evaluated": evaluated[indices].astype(np.uint8).tolist(), + "disagreement": ( + evaluated[indices] + & (prediction.ground_mask[indices] != ground_truth[indices]) + ) + .astype(np.uint8) + .tolist(), + "metrics": metrics, + "latency_ms": prediction.latency_ms, + "provider": dict(segmenter.identity), + "safety": { + "qualification_only": True, + "navigation_or_safety_accepted": False, + }, + } + preview_path = output_root / "preview.json" + _atomic_json(preview_path, preview) + report = { + "schema_version": GOOSE_GROUND_BENCHMARK_SCHEMA, + "identity_sha256": identity_sha256, + "identity": identity_document, + "ground_truth": { + "source": "GOOSE point-wise semantic labels", + "ground_categories": ["artificial_ground", "natural_ground"], + "void_excluded": True, + "evaluated_points": int(np.count_nonzero(evaluated)), + }, + "metrics": metrics, + "latency": { + "provider_ms": prediction.latency_ms, + "wall_ms": wall_latency_ms, + }, + "artifacts": { + "prediction_sha256": prediction_sha256, + "preview_sha256": _sha256_file(preview_path), + }, + "decision": { + "status": "diagnostic-baseline", + "promoted": False, + "reason": "one public frame does not qualify a production ground provider", + }, + } + _atomic_json(report_path, report) + return report + + +def _is_worker_dataset_root(root: Path) -> bool: + normalized = str(root).replace("\\", "/").rstrip("/").lower() + return normalized == "/mnt/d/ndc_missioncore/datasets" + + +def _binary_metrics( + prediction: np.ndarray[Any, Any], + ground_truth: np.ndarray[Any, Any], + evaluated: np.ndarray[Any, Any], +) -> dict[str, float | int]: + predicted = prediction & evaluated + truth = ground_truth & evaluated + true_positive = int(np.count_nonzero(predicted & truth)) + false_positive = int(np.count_nonzero(predicted & ~truth & evaluated)) + false_negative = int(np.count_nonzero(~predicted & truth)) + true_negative = int(np.count_nonzero(~predicted & ~truth & evaluated)) + precision = _ratio(true_positive, true_positive + false_positive) + recall = _ratio(true_positive, true_positive + false_negative) + return { + "true_positive": true_positive, + "false_positive": false_positive, + "false_negative": false_negative, + "true_negative": true_negative, + "precision": precision, + "recall": recall, + "f1": _ratio(2 * precision * recall, precision + recall), + "ground_iou": _ratio( + true_positive, + true_positive + false_positive + false_negative, + ), + "accuracy": _ratio( + true_positive + true_negative, + true_positive + true_negative + false_positive + false_negative, + ), + } + + +def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float: + return _ratio( + int(np.count_nonzero(prediction & truth)), + int(np.count_nonzero(truth)), + ) + + +def _ratio(numerator: float | int, denominator: float | int) -> float: + return float(numerator / denominator) if denominator else 0.0 + + +def _canonical_sha256(value: dict[str, Any]) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024**2), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None: + if path.exists(): + raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists") + with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".npz", delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + np.savez_compressed(temporary_path, **arrays) # type: ignore[arg-type] + with temporary_path.open("rb") as source: + os.fsync(source.fileno()) + os.replace(temporary_path, path) + except Exception: + temporary_path.unlink(missing_ok=True) + raise + + +def _atomic_json(path: Path, value: dict[str, Any]) -> None: + if path.exists(): + raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists") + encoded = ( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" + ) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + temporary.write(encoded) + temporary.flush() + os.fsync(temporary.fileno()) + os.replace(temporary_path, path) diff --git a/src/k1link/ground_segmentation.py b/src/k1link/ground_segmentation.py new file mode 100644 index 0000000..4c4d8d6 --- /dev/null +++ b/src/k1link/ground_segmentation.py @@ -0,0 +1,238 @@ +"""Provider-neutral, dependency-light LiDAR ground segmentation primitives.""" + +from __future__ import annotations + +import hashlib +import math +import time +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Protocol + +import numpy as np +import numpy.typing as npt + +BoolArray = npt.NDArray[np.bool_] + + +class GroundSegmentationError(ValueError): + """A ground provider or profile violates the shared segmentation contract.""" + + +@dataclass(frozen=True, slots=True) +class GroundBenchmarkProfile: + profile_id: str = "k1-vendor-map-ground-ab/v1" + pose_binding_threshold_ms: float = 100.0 + current_cell_size_m: float = 0.5 + current_local_radius_m: float = 2.5 + current_lower_percentile: float = 8.0 + current_maximum_below_ground_m: float = 0.25 + current_maximum_above_ground_m: float = 0.12 + current_minimum_local_points: int = 8 + patchwork_sensor_height_proxy_m: float = 0.0 + patchwork_map_vertical_origin_offset_m: float = 0.0 + patchwork_height_evidence: str = "missing" + patchwork_minimum_range_m: float = 0.1 + patchwork_maximum_range_m: float = 20.0 + + def __post_init__(self) -> None: + finite_values = ( + self.pose_binding_threshold_ms, + self.current_cell_size_m, + self.current_local_radius_m, + self.current_lower_percentile, + self.current_maximum_below_ground_m, + self.current_maximum_above_ground_m, + self.patchwork_sensor_height_proxy_m, + self.patchwork_map_vertical_origin_offset_m, + self.patchwork_minimum_range_m, + self.patchwork_maximum_range_m, + ) + if not all(math.isfinite(value) for value in finite_values): + raise GroundSegmentationError("Ground benchmark profile must be finite") + if ( + not 0 < self.pose_binding_threshold_ms <= 10_000 + or not 0 < self.current_cell_size_m <= 100 + or not 0 < self.current_local_radius_m <= 1_000 + or not 0 <= self.current_lower_percentile <= 100 + or not 0 <= self.current_maximum_below_ground_m <= 100 + or not 0 <= self.current_maximum_above_ground_m <= 100 + or not 1 <= self.current_minimum_local_points <= 1_000_000 + or not 0 <= self.patchwork_sensor_height_proxy_m <= 10 + or not -10 <= self.patchwork_map_vertical_origin_offset_m <= 10 + or not 0 <= self.patchwork_minimum_range_m < self.patchwork_maximum_range_m <= 1_000 + or self.patchwork_height_evidence + not in {"missing", "operator-estimated", "runtime-calibrated"} + ): + raise GroundSegmentationError("Ground benchmark profile is invalid") + if self.patchwork_height_evidence == "missing" and ( + self.patchwork_sensor_height_proxy_m != 0 + or self.patchwork_map_vertical_origin_offset_m != 0 + ): + raise GroundSegmentationError( + "Ground benchmark cannot apply height without height evidence" + ) + if ( + self.patchwork_height_evidence != "missing" + and self.patchwork_sensor_height_proxy_m <= 0 + ): + raise GroundSegmentationError( + "Ground benchmark height evidence requires a positive sensor height" + ) + + def to_dict(self) -> dict[str, object]: + return { + "schema_version": "missioncore.lidar-ground-benchmark-profile/v1", + "profile_id": self.profile_id, + "pose_binding": { + "basis": "nearest-recorded-host-monotonic-arrival", + "threshold_ms": self.pose_binding_threshold_ms, + }, + "current_baseline": { + "provider_id": "missioncore-local-percentile-ground/v1", + "derived_from": "local-ground-relative-object-support-v1", + "cell_size_m": self.current_cell_size_m, + "local_radius_m": self.current_local_radius_m, + "lower_percentile": self.current_lower_percentile, + "maximum_below_ground_m": self.current_maximum_below_ground_m, + "maximum_above_ground_m": self.current_maximum_above_ground_m, + "minimum_local_points": self.current_minimum_local_points, + }, + "candidate": { + "provider_id": "patchworkpp/v1.4.1", + "sensor_height_proxy_m": self.patchwork_sensor_height_proxy_m, + "map_vertical_origin_offset_m": self.patchwork_map_vertical_origin_offset_m, + "height_evidence": self.patchwork_height_evidence, + "minimum_range_m": self.patchwork_minimum_range_m, + "maximum_range_m": self.patchwork_maximum_range_m, + "enable_rnr": True, + "enable_rvpf": True, + "enable_tgr": True, + }, + "input_normalization": { + "current": "vendor-map-xyz", + "candidate": "best-effort-map-to-lidar-pose-inversion", + "physical_sensor_height_known": ( + self.patchwork_height_evidence == "runtime-calibrated" + ), + "sensor_scan_geometry_known": False, + }, + "authority": { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + + +DEFAULT_GROUND_BENCHMARK_PROFILE: Final = GroundBenchmarkProfile() + + +@dataclass(frozen=True, slots=True) +class GroundSegmentation: + ground_mask: BoolArray + assigned_mask: BoolArray + latency_ms: float + + +class GroundSegmenter(Protocol): + @property + def identity(self) -> Mapping[str, object]: ... + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ... + + +class LocalPercentileGroundSegmenter: + """Full-frame diagnostic extension of the existing E19 local ground heuristic.""" + + def __init__(self, profile: GroundBenchmarkProfile) -> None: + self.profile = profile + + @property + def identity(self) -> Mapping[str, object]: + return { + "provider_id": "missioncore-local-percentile-ground/v1", + "implementation_sha256": _sha256(Path(__file__).resolve(strict=True)), + "ground_truth": False, + } + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: + points = _xyzi(xyzi) + started = time.perf_counter_ns() + xyz = points[:, :3].astype(np.float64, copy=False) + cell_size = self.profile.current_cell_size_m + cell_keys = np.floor(xyz[:, :2] / cell_size).astype(np.int64) + unique_cells, inverse = np.unique(cell_keys, axis=0, return_inverse=True) + order = np.argsort(inverse, kind="stable") + counts = np.bincount(inverse, minlength=unique_cells.shape[0]) + offsets = np.concatenate(([0], np.cumsum(counts))) + cell_lookup = { + (int(cell[0]), int(cell[1])): cell_index + for cell_index, cell in enumerate(unique_cells) + } + neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1 + ground = np.zeros(points.shape[0], dtype=np.bool_) + global_ground_z = float(np.percentile(xyz[:, 2], self.profile.current_lower_percentile)) + radius_squared = self.profile.current_local_radius_m**2 + for cell_index in range(unique_cells.shape[0]): + point_indices = order[offsets[cell_index] : offsets[cell_index + 1]] + if point_indices.size == 0: + continue + center_xy = np.median(xyz[point_indices, :2], axis=0) + cell_x, cell_y = unique_cells[cell_index] + neighbor_slices: list[npt.NDArray[np.int64]] = [] + for delta_x in range(-neighbor_span, neighbor_span + 1): + for delta_y in range(-neighbor_span, neighbor_span + 1): + neighbor_cell_index = cell_lookup.get( + (int(cell_x + delta_x), int(cell_y + delta_y)) + ) + if neighbor_cell_index is None: + continue + neighbor_slices.append( + order[ + offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1] + ] + ) + local_indices = np.concatenate(neighbor_slices) + local_xyz = xyz[local_indices] + delta_xy = local_xyz[:, :2] - center_xy + local = local_xyz[ + np.einsum("ij,ij->i", delta_xy, delta_xy) <= radius_squared, + 2, + ] + ground_z = ( + float(np.percentile(local, self.profile.current_lower_percentile)) + if local.size >= self.profile.current_minimum_local_points + else global_ground_z + ) + z = xyz[point_indices, 2] + ground[point_indices] = ( + z >= ground_z - self.profile.current_maximum_below_ground_m + ) & (z <= ground_z + self.profile.current_maximum_above_ground_m) + latency_ms = (time.perf_counter_ns() - started) / 1_000_000 + return GroundSegmentation( + ground_mask=ground, + assigned_mask=np.ones(points.shape[0], dtype=np.bool_), + latency_ms=latency_ms, + ) + + +def _xyzi(value: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]: + points = np.asarray(value) + if ( + points.dtype != np.dtype(np.float32) + or points.ndim != 2 + or points.shape[1] != 4 + or points.shape[0] == 0 + or not np.isfinite(points).all() + ): + raise GroundSegmentationError("Ground input must be a non-empty finite float32 XYZI matrix") + return points + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024**2), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 74e76a9..2462b3d 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -415,6 +415,15 @@ app.include_router( / "lidar-ground-v1" / "benchmarks" ), + dataset_admission_provider=lambda: ( + REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "admission.json" + ), + dataset_preview_provider=lambda: ( + REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "preview.json" + ), + dataset_ground_preview_provider=lambda: ( + REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json" + ), ) ) diff --git a/src/k1link/web/lidar_api.py b/src/k1link/web/lidar_api.py index 05a98c4..6880bba 100644 --- a/src/k1link/web/lidar_api.py +++ b/src/k1link/web/lidar_api.py @@ -20,7 +20,15 @@ from k1link.compute import ( lidar_pack_catalog_item, lidar_pack_detail, ) -from k1link.datasets import dataset_gateway_catalog +from k1link.datasets import ( + DatasetAdmissionError, + configured_dataset_admission_manifest, + configured_dataset_ground_preview, + configured_dataset_preview, + dataset_gateway_catalog, + read_dataset_ground_preview, + read_dataset_native_scan_preview, +) LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1" LIDAR_GROUND_CATALOG_SCHEMA: Final = "missioncore.lidar-ground-benchmark-catalog/v1" @@ -29,6 +37,7 @@ _PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$") _BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$") _FIELD_REVIEW_ID = re.compile(r"^lidar-field-review-[a-f0-9]{64}$") RootProvider = Callable[[], Path | None] +DatasetArtifactProvider = Callable[[], Path | None] def configured_lidar_replay_root() -> Path | None: @@ -51,12 +60,49 @@ def build_lidar_router( root_provider: RootProvider = configured_lidar_replay_root, ground_root_provider: RootProvider = configured_lidar_ground_root, field_review_root_provider: RootProvider = configured_lidar_field_review_root, + dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest, + dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview, + dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview, ) -> APIRouter: router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"]) @router.get("/dataset-gateway") def get_dataset_gateway() -> dict[str, object]: - return dataset_gateway_catalog() + return dataset_gateway_catalog( + admission_manifest_path=dataset_admission_provider(), + ) + + @router.get("/dataset-gateway/preview") + def get_dataset_gateway_preview() -> dict[str, Any]: + path = dataset_preview_provider() + if path is None or not path.is_file(): + raise HTTPException( + status_code=404, + detail="Первый размеченный native scan ещё не импортирован.", + ) + try: + return read_dataset_native_scan_preview(path) + except DatasetAdmissionError as exc: + raise HTTPException( + status_code=500, + detail="Preview датасета не прошло проверку целостности.", + ) from exc + + @router.get("/dataset-gateway/ground-comparison") + def get_dataset_gateway_ground_comparison() -> dict[str, Any]: + path = dataset_ground_preview_provider() + if path is None or not path.is_file(): + raise HTTPException( + status_code=404, + detail="Ground baseline ещё не сравнивался с GOOSE-разметкой.", + ) + try: + return read_dataset_ground_preview(path) + except DatasetAdmissionError as exc: + raise HTTPException( + status_code=500, + detail="Ground comparison не прошло проверку целостности.", + ) from exc @router.get("/replay-packs") def list_lidar_replay_packs( diff --git a/tests/test_dataset_gateway.py b/tests/test_dataset_gateway.py index 3ef6d48..56c5d3d 100644 --- a/tests/test_dataset_gateway.py +++ b/tests/test_dataset_gateway.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +import zipfile from pathlib import Path import numpy as np @@ -8,10 +10,18 @@ from fastapi import APIRouter from fastapi.routing import APIRoute from k1link.datasets import ( + DatasetAdmissionError, DatasetFrameError, dataset_gateway_catalog, + goose_admission, + goose_benchmark, + read_dataset_admission_manifest, + read_dataset_ground_preview, + read_dataset_native_scan_preview, read_semantic_kitti_frame, ) +from k1link.datasets.goose_admission import admit_goose_validation +from k1link.datasets.goose_benchmark import benchmark_goose_current_ground from k1link.web.lidar_api import build_lidar_router @@ -97,7 +107,7 @@ def test_dataset_gateway_api_is_read_only_and_path_free( route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway") response = route() # type: ignore[operator] - assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v1" + assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2" assert response["access"] == "read-only" assert response["storage"]["admitted"] is True assert response["storage"]["path_exposed"] is False @@ -105,3 +115,251 @@ def test_dataset_gateway_api_is_read_only_and_path_free( response["storage"]["required_wsl_root"], # type: ignore[index] "", ) + + +def _downloading_manifest() -> dict[str, object]: + return { + "schema_version": "missioncore.dataset-admission/v1", + "source_id": "goose-3d/v2025-08-22", + "observed_at_utc": "2026-07-25T10:00:00Z", + "status": "downloading", + "storage": { + "policy": "worker-d-only", + "admitted": True, + "canonical_root": True, + "path_exposed": False, + }, + "archive": { + "filename": "goose_3d_val.zip", + "source_url": "https://goose-dataset.de/storage/goose_3d_val.zip", + "bytes_transferred": 512, + "total_bytes": 1024, + "size_bytes": None, + "sha256": None, + "integrity": "pending", + "vendor_checksum_available": False, + }, + "license": { + "spdx": "CC-BY-SA-4.0", + "artifact_present": False, + }, + "frame": None, + "next_action": "complete-download-and-admit", + } + + +def test_dataset_gateway_uses_worker_manifest_instead_of_static_download_state( + tmp_path: Path, +) -> None: + manifest_path = tmp_path / "admission.json" + manifest_path.write_text(json.dumps(_downloading_manifest()), encoding="utf-8") + + catalog = dataset_gateway_catalog( + tmp_path / "not-worker-d", + admission_manifest_path=manifest_path, + ) + + assert catalog["storage"]["admitted"] is True # type: ignore[index] + assert catalog["storage"]["attestation"] == "worker-manifest" # type: ignore[index] + source = catalog["sources"][0] # type: ignore[index] + assert source["admission"]["status"] == "downloading" # type: ignore[index] + assert source["admission"]["archive"]["bytes_transferred"] == 512 # type: ignore[index] + assert catalog["next_action"] == "complete-download-and-admit" + + +def test_dataset_admission_manifest_fails_closed_on_forged_storage( + tmp_path: Path, +) -> None: + manifest = _downloading_manifest() + manifest["storage"]["path_exposed"] = True # type: ignore[index] + path = tmp_path / "admission.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(DatasetAdmissionError, match="storage admission"): + read_dataset_admission_manifest(path) + + +def test_goose_admission_extracts_one_aligned_frame_and_bounded_preview( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "datasets" + archive = root / "goose-3d/v2025-08-22/archives/goose_3d_val.zip" + archive.parent.mkdir(parents=True) + points = np.asarray( + [ + [1.0, 2.0, 3.0, 0.25], + [-4.0, 5.0, 0.5, 0.75], + [2.0, 1.0, -0.2, 0.5], + ], + dtype=" None: + preview_path = tmp_path / "preview.json" + preview_path.write_text( + json.dumps( + { + "schema_version": "missioncore.dataset-native-scan-preview/v1", + "source_id": "goose-3d/v2025-08-22", + "frame_id": "frame-1", + "representation": "native-scan", + "sampling": "deterministic-even-index", + "source_point_count": 1, + "point_count": 1, + "points_xyz_m": [[1.0, 2.0, 3.0]], + "remission_0_to_255": [128], + "semantic_label_ids": [23], + "semantic_rgb_0_to_255": [255, 47, 128], + "ground_truth_ground": [1], + "classes": [ + { + "label_id": 23, + "class_name": "asphalt", + "hex": "#ff2f80", + "challenge_category_id": 2, + "challenge_category_name": "artificial_ground", + } + ], + "safety": { + "visualization_only": True, + "navigation_or_safety_accepted": False, + }, + } + ), + encoding="utf-8", + ) + router = build_lidar_router( + dataset_admission_provider=lambda: None, + dataset_preview_provider=lambda: preview_path, + ) + route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview") + + response = route() # type: ignore[operator] + + assert response["frame_id"] == "frame-1" + + +def test_goose_current_ground_is_scored_against_independent_labels( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "datasets" + digest = "a" * 64 + frame_id = "frame-1" + install = root / f"goose-3d/v2025-08-22/installs/{digest}" + frame_root = install / f"frames/{frame_id}" + frame_root.mkdir(parents=True) + np.asarray( + [ + [-1.0, 0.0, 0.00, 0.1], + [0.0, 0.0, 0.02, 0.2], + [1.0, 0.0, 0.01, 0.3], + [0.0, 0.2, 1.50, 0.4], + ], + dtype=" None: np.testing.assert_array_equal(xyzi, unchanged) +def test_local_percentile_grid_index_preserves_exact_radius_result() -> None: + random = np.random.default_rng(42) + xyzi = random.normal(size=(240, 4)).astype(np.float32) + xyzi[:, :2] *= 4 + profile = GroundBenchmarkProfile( + current_cell_size_m=0.6, + current_local_radius_m=1.7, + current_lower_percentile=11, + current_maximum_below_ground_m=0.2, + current_maximum_above_ground_m=0.18, + current_minimum_local_points=5, + ) + + actual = LocalPercentileGroundSegmenter(profile=profile).segment(xyzi) + + xyz = xyzi[:, :3].astype(np.float64) + cell_keys = np.floor(xyz[:, :2] / profile.current_cell_size_m).astype(np.int64) + unique_cells, inverse = np.unique(cell_keys, axis=0, return_inverse=True) + expected = np.zeros(xyzi.shape[0], dtype=np.bool_) + global_ground_z = float(np.percentile(xyz[:, 2], profile.current_lower_percentile)) + for cell_index in range(unique_cells.shape[0]): + point_indices = np.flatnonzero(inverse == cell_index) + center_xy = np.median(xyz[point_indices, :2], axis=0) + delta_xy = xyz[:, :2] - center_xy + local = xyz[ + np.einsum("ij,ij->i", delta_xy, delta_xy) + <= profile.current_local_radius_m**2, + 2, + ] + ground_z = ( + float(np.percentile(local, profile.current_lower_percentile)) + if local.size >= profile.current_minimum_local_points + else global_ground_z + ) + z = xyz[point_indices, 2] + expected[point_indices] = ( + z >= ground_z - profile.current_maximum_below_ground_m + ) & (z <= ground_z + profile.current_maximum_above_ground_m) + + np.testing.assert_array_equal(actual.ground_mask, expected) + + def test_ground_benchmark_is_immutable_diagnostic_evidence(tmp_path: Path) -> None: replay = _Replay() output = build_lidar_ground_benchmark(