{previewError ?? "Проверяем point alignment и разметку."}
)} + {comparison ? ( ++ Patchwork++ использует опубликованную высоту VLS-128{" "} + {comparison.inputProfile.sensorHeightM.toFixed(2)} м; это + algorithm-specific admission, а не полный vehicle TF и не + normalized scan. +
+ ) : null} ) : null} diff --git a/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx b/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx index 3899efa..113074d 100644 --- a/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx +++ b/apps/control-station/src/workspaces/LidarGroundPointCloud.tsx @@ -7,6 +7,7 @@ export type LidarGroundViewMode = | "current" | "candidate" | "disagreement" + | "candidate-disagreement" | "semantic" | "ground-truth"; @@ -21,6 +22,7 @@ export interface LidarGroundPointCloudFrame { candidateGround: number[]; candidateAssigned: number[]; disagreement: number[]; + candidateDisagreement?: number[]; groundTruthGround?: number[]; }; } @@ -119,6 +121,15 @@ function frameColors( disagreement ? 0.31 : 0.29, disagreement ? 0.22 : 0.35, ); + } else if (mode === "candidate-disagreement") { + const disagreement = frame.masks.candidateDisagreement?.[index] === 1; + setRgb( + colors, + offset, + disagreement ? 1 : 0.24, + disagreement ? 0.31 : 0.29, + disagreement ? 0.22 : 0.35, + ); } else { setRgb(colors, offset, candidate ? 0.24 : 0.29, candidate ? 0.84 : 0.36, 0.43); } diff --git a/apps/control-station/test/datasetGateway.test.mjs b/apps/control-station/test/datasetGateway.test.mjs index 09ecc06..51c7734 100644 --- a/apps/control-station/test/datasetGateway.test.mjs +++ b/apps/control-station/test/datasetGateway.test.mjs @@ -194,36 +194,97 @@ test("decodes one bounded point-aligned native-scan preview", async () => { ); }); -test("decodes a point-aligned current-vs-ground-truth comparison", async () => { +test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => { const payload = { - schema_version: "missioncore.dataset-ground-comparison-preview/v1", + schema_version: "missioncore.dataset-ground-comparison-preview/v2", source_id: "goose-3d/v2025-08-22", frame_id: "frame-1", sampling: "deterministic-even-index", point_count: 2, current_ground: [1, 1], + patchwork_ground: [1, 0], + patchwork_assigned: [1, 1], ground_truth_ground: [1, 0], evaluated: [1, 1], - disagreement: [0, 1], + current_disagreement: [0, 1], + patchwork_disagreement: [0, 0], metrics: { - true_positive: 1, - false_positive: 1, - false_negative: 0, - true_negative: 0, - precision: 0.5, - recall: 1, - f1: 2 / 3, - ground_iou: 0.5, - accuracy: 0.5, - artificial_ground_recall: 1, - natural_ground_recall: 0, - obstacle_non_ground_recall: 1, + current: { + true_positive: 1, + false_positive: 1, + false_negative: 0, + true_negative: 0, + precision: 0.5, + recall: 1, + f1: 2 / 3, + ground_iou: 0.5, + accuracy: 0.5, + artificial_ground_recall: 1, + natural_ground_recall: 0, + obstacle_non_ground_recall: 1, + }, + patchworkpp: { + true_positive: 1, + false_positive: 0, + false_negative: 0, + true_negative: 1, + precision: 1, + recall: 1, + f1: 1, + ground_iou: 1, + accuracy: 1, + artificial_ground_recall: 1, + natural_ground_recall: 1, + obstacle_non_ground_recall: 1, + }, }, - latency_ms: 12.5, - provider: { - provider_id: "missioncore-local-percentile-ground/v1", - implementation_sha256: "a".repeat(64), - ground_truth: false, + latency_ms: { + current: 12.5, + patchworkpp: 0.4, + }, + patchwork_assigned_fraction: 1, + providers: { + current: { + provider_id: "missioncore-local-percentile-ground/v1", + implementation_sha256: "a".repeat(64), + ground_truth: false, + }, + patchworkpp: { + provider_id: "patchworkpp/v1.4.1", + source_url: "https://github.com/url-kaist/patchwork-plusplus", + source_tag: "v1.4.1", + source_commit: "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c", + binding_version: "0.0.1", + binary_sha256: "b".repeat(64), + platform: "linux", + machine: "x86_64", + ground_truth: false, + }, + }, + input_profile: { + schema_version: "missioncore.goose-patchwork-profile/v1", + source_id: "goose-3d/v2025-08-22", + representation: "native-scan", + sensor_frame: { + frame_id: "sensor/lidar/vls128_roof", + handedness: "right", + x: "forward", + y: "left", + z: "up", + one_revolution: true, + }, + height: { + base_link_above_ground_m: 0.64, + lidar_above_base_link_m: 1.6, + sensor_above_ground_m: 2.24, + evidence: "published-dimensioned-schematic", + }, + scope: { + patchworkpp_eligible: true, + normalized_scan_produced: false, + complete_vehicle_transform_known: false, + deskew_claimed: false, + }, }, safety: { qualification_only: true, @@ -231,15 +292,23 @@ test("decodes a point-aligned current-vs-ground-truth comparison", async () => { }, }; const parsed = parseDatasetGroundComparison(payload); - assert.equal(parsed.metrics.groundIou, 0.5); - assert.deepEqual(parsed.disagreement, [0, 1]); + assert.equal(parsed.metrics.current.groundIou, 0.5); + assert.equal(parsed.metrics.patchworkpp.groundIou, 1); + assert.deepEqual(parsed.patchworkDisagreement, [0, 0]); + assert.equal(parsed.inputProfile.sensorHeightM, 2.24); const fetched = await fetchDatasetGroundComparison({ fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }), }); - assert.equal(fetched.latencyMs, 12.5); + assert.equal(fetched.latencyMs.patchworkpp, 0.4); - payload.disagreement.pop(); + payload.patchwork_disagreement.pop(); + assert.throws( + () => parseDatasetGroundComparison(payload), + DatasetGatewayContractError, + ); + payload.patchwork_disagreement.push(0); + payload.input_profile.height.sensor_above_ground_m = 2.25; assert.throws( () => parseDatasetGroundComparison(payload), DatasetGatewayContractError, diff --git a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md index 0a53fc4..0c078a8 100644 --- a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md +++ b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md @@ -2,7 +2,7 @@ Date: 2026-07-25 Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete; -Dataset Gateway S0 implemented, first real GOOSE import pending worker D storage +Dataset Gateway first-frame Current/Patchwork++ A/B complete Scope: real scanner records, replay and future live shadow processing Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering @@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later domain adaptation after a public baseline proves that the pipeline and metric harness work. -### L2.5 — Dataset Gateway — S0 complete, real import pending +### L2.5 — Dataset Gateway — first-frame A/B complete - [x] Define separate `native-scan`, `normalized-scan` and `rolling-local-map` representations. @@ -304,13 +304,16 @@ harness work. per-point time and line/ring fields. - [x] Require operator-admitted storage under `D:\NDC_MISSIONCORE\datasets`. -- [ ] Configure the worker dataset root and record disk/resource baseline. -- [ ] Download only the 3.3 GB GOOSE validation archive first and record its +- [x] Configure the worker dataset root and record disk/resource baseline. +- [x] Download only the 3.3 GB GOOSE validation archive first and record its hash/license/provenance. -- [ ] Show one real labeled revolution in React with native remission and +- [x] Show one real labeled revolution in React with native remission and ground-truth superclass coloring. -- [ ] Admit an explicit frame/mounting profile before normalization. -- [ ] Run current ground and Patchwork++ against GOOSE ground truth. +- [x] Admit the published Patchwork-specific axes/height profile without + claiming full vehicle TF, deskew or a general normalized scan. +- [x] Run current ground and Patchwork++ against the same GOOSE native scan and + ground truth; keep the result one-frame diagnostic. +- [ ] Expand Current/Patchwork++ qualification to the validation split. - [ ] Add named range/FOV/density/noise/dropout degradation profiles without overwriting the native frame. @@ -387,8 +390,8 @@ The near-term value is not a prettier point cloud: - hardware selection becomes evidence-driven: a future vehicle LiDAR is accepted by its timing/fields/profile, not by vendor marketing. -The highest-value immediate work is the worker-side GOOSE validation import, -the first labeled native scan in React and an accuracy-bearing ground A/B. -LiDAR-native detection follows on the same gateway. Nvblox and alternative -SLAM remain later because their timing, pose and scan-geometry gates are not yet -satisfied. +The highest-value immediate work is now the validation-split +Current/Patchwork++ gate plus named range/FOV/density/noise/dropout degradation +profiles. LiDAR-native detection follows on the same gateway. Nvblox and +alternative SLAM remain later because their timing, pose and scan-geometry +gates are not yet satisfied. diff --git a/docs/14_LIDAR_DATASET_GATEWAY.md b/docs/14_LIDAR_DATASET_GATEWAY.md index 73ccbbe..366f53f 100644 --- a/docs/14_LIDAR_DATASET_GATEWAY.md +++ b/docs/14_LIDAR_DATASET_GATEWAY.md @@ -39,6 +39,12 @@ Implemented now: `Открыть` action; - one admitted GOOSE validation frame with source colors, normalized remission and an independent ground-truth view; +- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile: + sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native + one-revolution scan and explicit absence of deskew/full vehicle TF claims; +- a reproducible Current/Patchwork++ A/B artifact with point-aligned masks, + disagreement views, independent-label metrics, latency and exact provider + identities; - a separate **Парк → Диагностика LiDAR** surface containing only real sensor recordings and their operational evidence. @@ -152,11 +158,13 @@ time; TTL/dynamic filtering prevents stale ghosts. 4. [Done] Import one labeled frame and expose it in React as `native-scan`. 5. [Done] Show native remission, the original semantic palette and ground-truth superclass coloring. -6. Add a declared GOOSE frame/mounting profile and produce `normalized-scan`. -7. [Current baseline done; Patchwork++ blocked on mounting evidence] Run the - current ground heuristic and Patchwork++ against independent labels. +6. [Patchwork-specific profile done; general normalization still pending] Admit + the published VLS-128 axes and physical height without claiming a complete + numeric vehicle TF or deskewed `normalized-scan`. +7. [One-frame A/B done] Run the current ground heuristic and pinned + Patchwork++ against the same native scan and independent labels. 8. Add sensor-degradation profiles for range, FOV, density, noise and dropout. -9. Only after the one-frame contract passes, expand to the validation split and +9. Expand the A/B to the validation split; only after that qualification gate, add a rolling-map sequence with localization evidence. ## Acceptance checklist @@ -172,9 +180,11 @@ time; TTL/dynamic filtering prevents stale ghosts. - [x] Worker D root configured. - [x] GOOSE validation archive hash recorded. - [x] First real labeled frame visible in React. -- [ ] Coordinate and mounting profile admitted. +- [x] Patchwork-specific axes and physical-height profile admitted. +- [ ] Complete vehicle transform and general `normalized-scan` admitted. - [x] Current local-percentile baseline measured against ground truth. -- [ ] Patchwork++ accuracy measured against ground truth. +- [x] Patchwork++ one-frame accuracy measured against ground truth. +- [ ] Current/Patchwork++ validation-split gate qualified. - [ ] Sensor-degradation matrix qualified. - [ ] Rolling local map with pose/TTL/dynamic policy qualified. @@ -229,10 +239,54 @@ previous exact-radius result. This removes the previous all-points scan for every occupied cell, but the measured latency still classifies it as a diagnostic baseline rather than an onboard candidate. -Patchwork++ is intentionally not scored yet. The validation ZIP contains XYZI, -labels, mapping, LICENSE and CHANGELOG but no numeric TF/mounting calibration. -GOOSE documents the VLS-128 as a roof LiDAR and publishes a separate MuCAR-3 TF -tree, but the graph image alone is not physical-height evidence. The next gate -is to admit the numeric transform from `base_link_ground` to -`sensor/lidar/vls128_roof`, declare the source axis convention, and only then -run Patchwork++. +## First Patchwork++ A/B result + +GOOSE's dimensioned MuCAR-3 schematic provides the two vertical dimensions +needed by this algorithm-specific gate: `base_link` is `0.64 m` above ground +and the VLS-128 optical center is `1.60 m` above `base_link`. The admitted +Patchwork++ height is therefore `2.24 m`. The same published schematic declares +the sensor axes as `x` forward, `y` left and `z` up. An independent fit to +GOOSE-labeled near-field ground observed a `-2.18 .. -2.14 m` intercept; this +was a non-calibrating cross-check, not the source of the height. + +This narrowly admits the input required by Patchwork++ on the native +sensor-centric scan. It does **not** claim a complete numeric vehicle transform, +per-point timing, deskew or a general `normalized-scan`. + +Official Patchwork++ `v1.4.1`, source commit +`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`, was run against the same first +frame and independent labels as Current: + +| Metric | Current | Patchwork++ | +| --- | ---: | ---: | +| Ground IoU | `49.6165%` | `60.3702%` | +| Precision | `73.5892%` | `72.6612%` | +| Recall | `60.3659%` | `78.1130%` | +| F1 | `66.3249%` | `75.2886%` | +| Artificial-ground recall | `95.2364%` | `98.9688%` | +| Natural-ground recall | `49.4516%` | `71.5853%` | +| Obstacle non-ground recall | `79.6145%` | `92.1729%` | +| Worker latency | `3966.53 ms` | `15.89 ms` | + +Reproducibility pins: + +- benchmark identity: + `2a6d05f54a9e2ac727d9c850c1133f2fb539bfeddbd97c07cfd198ccb239162c`; +- full point-aligned prediction SHA-256: + `190f455c6e47911921b7f6454913e1bb2d0b8b5814ee2e34d7b63295afddc7ef`; +- bounded browser preview SHA-256: + `e23ca573103faf931523415e6c263248737eb6600f482b63de685b725bbc28c3`; +- Patchwork++ binary SHA-256: + `be8038b2098c83fe53841aa8ae19e362910e9056fe0ee7b9304c1ee5c5941094`. + +Patchwork++ wins this frame by `10.75` percentage points of Ground IoU, raises +natural-ground recall by `22.13` points and is roughly `250x` faster in this +run. The result is deliberately `one-frame-diagnostic`: it makes Patchwork++ +the candidate for validation-split and degradation qualification, not an +accepted navigation or safety provider. + +Primary source evidence: + +- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/); +- [GOOSE paper, Figure 3](https://arxiv.org/pdf/2310.16788); +- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1). diff --git a/docs/adr/0021-dataset-gateway-representation-boundary.md b/docs/adr/0021-dataset-gateway-representation-boundary.md index 0529a72..6cbe9f8 100644 --- a/docs/adr/0021-dataset-gateway-representation-boundary.md +++ b/docs/adr/0021-dataset-gateway-representation-boundary.md @@ -87,24 +87,34 @@ unavailable rather than inventing timestamps. - Sensor adaptation may change range, FOV, point density, noise and dropout for robustness experiments, but it cannot recreate lost timestamps, occlusions or material response. -- Patchwork++ becomes eligible for a real quality gate only on a sensor-centric - scan with declared scan geometry and physical mounting height plus - independent labels. +- Patchwork++ becomes eligible for an algorithm-specific quality gate on a + sensor-centric scan with declared axes, physical mounting height and + independent labels. That narrow admission does not itself create a general + `normalized-scan`. - Stable operator visualization is owned by rolling-map policy, not by the ground classifier. - Public labels may qualify an algorithm independently of the production sensor. The first GOOSE frame exposed a `49.62%` Ground IoU and only `49.45%` natural-ground recall for the current local-percentile baseline; these are diagnostic results, not production promotion. -- A dataset label contract does not imply a mounting contract. Patchwork++ - remains blocked until the numeric GOOSE roof-LiDAR transform and physical - height are admitted from source evidence. +- A dataset label contract does not imply a mounting contract. For MuCAR-3, + GOOSE's published dimensioned schematic admits the Patchwork-specific + `2.24 m` sensor height (`0.64 + 1.60 m`) and `x-forward / y-left / z-up` + axes. The complete numeric vehicle transform, deskew and general + `normalized-scan` remain unavailable. +- On the first independently labeled frame, pinned Patchwork++ `v1.4.1` + achieved `60.37%` Ground IoU, `71.59%` natural-ground recall and `15.89 ms` + worker latency versus Current's `49.62%`, `49.45%` and `3966.53 ms`. + This is a one-frame diagnostic candidate decision, not production promotion. ## Primary references - [GOOSE dataset structure](https://goose-dataset.de/docs/dataset-structure/) - [GOOSE setup and archive sizes](https://goose-dataset.de/docs/setup/) - [GOOSE 3D challenge ontology](https://goose-dataset.de/docs/3d-semantic-segmentation-challenge/) +- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/) +- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788) +- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1) - [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2) - [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox) - [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html) diff --git a/src/k1link/compute/lidar_ground.py b/src/k1link/compute/lidar_ground.py index 9d16e55..2a51c96 100644 --- a/src/k1link/compute/lidar_ground.py +++ b/src/k1link/compute/lidar_ground.py @@ -1,18 +1,14 @@ from __future__ import annotations import hashlib -import importlib import json import math import os -import platform import re import shutil -import time from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path -from types import ModuleType from typing import Any, Final import numpy as np @@ -25,9 +21,21 @@ from k1link.ground_segmentation import ( GroundSegmenter, LocalPercentileGroundSegmenter, ) +from k1link.ground_segmentation import ( + PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT, +) +from k1link.ground_segmentation import ( + PATCHWORKPP_SOURCE_TAG as PATCHWORKPP_SOURCE_TAG, +) +from k1link.ground_segmentation import ( + PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL, +) from k1link.ground_segmentation import ( GroundSegmentationError as LidarGroundError, ) +from k1link.ground_segmentation import ( + PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter, +) from .lidar_contract import LidarContractError, sensor_frame_xyzi from .lidar_replay import LidarReplayPackV2 @@ -42,107 +50,9 @@ LIDAR_GROUND_MANIFEST_NAME: Final = "manifest.json" LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz" MAX_GROUND_FRAME_POINTS: Final = 200_000 -PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus" -PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1" -PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c" - _BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$") _ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$") _SHA256 = re.compile(r"^[a-f0-9]{64}$") -_GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$") - - -class PatchworkPPGroundSegmenter: - """Runtime-only adapter for the pinned official Patchwork++ Python binding.""" - - def __init__( - self, - module: ModuleType, - profile: GroundBenchmarkProfile, - *, - source_commit: str = PATCHWORKPP_SOURCE_COMMIT, - source_tag: str = PATCHWORKPP_SOURCE_TAG, - ) -> None: - if _GIT_SHA1.fullmatch(source_commit) is None: - raise LidarGroundError("Patchwork++ source commit is invalid") - if source_tag != PATCHWORKPP_SOURCE_TAG: - raise LidarGroundError("Patchwork++ source tag is not admitted") - module_path_value = getattr(module, "__file__", None) - if not isinstance(module_path_value, str): - raise LidarGroundError("Patchwork++ module has no verifiable binary") - module_path = Path(module_path_value).resolve(strict=True) - params = module.Parameters() - params.sensor_height = profile.patchwork_sensor_height_proxy_m - params.min_range = profile.patchwork_minimum_range_m - params.max_range = profile.patchwork_maximum_range_m - params.enable_RNR = True - params.enable_RVPF = True - params.enable_TGR = True - params.verbose = False - self._estimator = module.patchworkpp(params) - self._identity = { - "provider_id": "patchworkpp/v1.4.1", - "source_url": PATCHWORKPP_SOURCE_URL, - "source_tag": source_tag, - "source_commit": source_commit, - "binding_version": str(getattr(module, "__version__", "unknown")), - "binary_sha256": _sha256(module_path), - "platform": platform.system().lower(), - "machine": platform.machine().lower(), - "ground_truth": False, - } - - @classmethod - def load( - cls, - profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, - *, - module_name: str = "pypatchworkpp", - source_commit: str = PATCHWORKPP_SOURCE_COMMIT, - source_tag: str = PATCHWORKPP_SOURCE_TAG, - ) -> PatchworkPPGroundSegmenter: - try: - module = importlib.import_module(module_name) - except ImportError as exc: - raise LidarGroundError("Pinned Patchwork++ Python binding is unavailable") from exc - return cls( - module, - profile, - source_commit=source_commit, - source_tag=source_tag, - ) - - @property - def identity(self) -> Mapping[str, object]: - return self._identity - - def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: - points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32) - started = time.perf_counter_ns() - self._estimator.estimateGround(points) - latency_ms = (time.perf_counter_ns() - started) / 1_000_000 - ground_indices = np.asarray( - self._estimator.getGroundIndices(), - dtype=np.int64, - ).reshape((-1,)) - nonground_indices = np.asarray( - self._estimator.getNongroundIndices(), - dtype=np.int64, - ).reshape((-1,)) - _indices(ground_indices, points.shape[0], "Patchwork++ ground") - _indices(nonground_indices, points.shape[0], "Patchwork++ non-ground") - if np.intersect1d(ground_indices, nonground_indices).size: - raise LidarGroundError("Patchwork++ assigned one point twice") - ground = np.zeros(points.shape[0], dtype=np.bool_) - assigned = np.zeros(points.shape[0], dtype=np.bool_) - ground[ground_indices] = True - assigned[ground_indices] = True - assigned[nonground_indices] = True - return GroundSegmentation( - ground_mask=ground, - assigned_mask=assigned, - latency_ms=latency_ms, - ) class LidarGroundBenchmarkV1: diff --git a/src/k1link/datasets/__init__.py b/src/k1link/datasets/__init__.py index 4506af9..57893ed 100644 --- a/src/k1link/datasets/__init__.py +++ b/src/k1link/datasets/__init__.py @@ -15,9 +15,17 @@ from k1link.datasets.gateway import ( read_semantic_kitti_frame, ) from k1link.datasets.goose_benchmark import ( + GOOSE_GROUND_AB_BENCHMARK_SCHEMA, + GOOSE_GROUND_AB_PREVIEW_SCHEMA, GOOSE_GROUND_BENCHMARK_SCHEMA, GOOSE_GROUND_PREVIEW_SCHEMA, benchmark_goose_current_ground, + benchmark_goose_patchwork_ground, +) +from k1link.datasets.goose_profile import ( + DEFAULT_GOOSE_PATCHWORK_PROFILE, + GOOSE_PATCHWORK_PROFILE_SCHEMA, + GoosePatchworkProfile, ) __all__ = [ @@ -25,9 +33,15 @@ __all__ = [ "DatasetAdmissionError", "DatasetFrameError", "DatasetPointFrame", + "DEFAULT_GOOSE_PATCHWORK_PROFILE", + "GOOSE_GROUND_AB_BENCHMARK_SCHEMA", + "GOOSE_GROUND_AB_PREVIEW_SCHEMA", "GOOSE_GROUND_BENCHMARK_SCHEMA", "GOOSE_GROUND_PREVIEW_SCHEMA", + "GOOSE_PATCHWORK_PROFILE_SCHEMA", + "GoosePatchworkProfile", "benchmark_goose_current_ground", + "benchmark_goose_patchwork_ground", "configured_dataset_admission_manifest", "configured_dataset_ground_preview", "configured_dataset_preview", diff --git a/src/k1link/datasets/cli.py b/src/k1link/datasets/cli.py index c75f138..c9f34f8 100644 --- a/src/k1link/datasets/cli.py +++ b/src/k1link/datasets/cli.py @@ -9,7 +9,10 @@ from typing import Annotated import typer from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation -from k1link.datasets.goose_benchmark import benchmark_goose_current_ground +from k1link.datasets.goose_benchmark import ( + benchmark_goose_current_ground, + benchmark_goose_patchwork_ground, +) app = typer.Typer( add_completion=False, @@ -63,5 +66,22 @@ def benchmark_goose_current_ground_command( typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True)) +@app.command("benchmark-goose-patchwork-ground") +def benchmark_goose_patchwork_ground_command( + dataset_root: Annotated[ + Path, + typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True), + ], +) -> None: + """Compare current and pinned Patchwork++ providers against GOOSE labels.""" + + try: + report = benchmark_goose_patchwork_ground(dataset_root) + except (GooseAdmissionError, ValueError) as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True)) + + if __name__ == "__main__": app() diff --git a/src/k1link/datasets/gateway.py b/src/k1link/datasets/gateway.py index a237920..9d3ee73 100644 --- a/src/k1link/datasets/gateway.py +++ b/src/k1link/datasets/gateway.py @@ -16,10 +16,12 @@ from typing import Any, Final, Literal import numpy as np import numpy.typing as npt +from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE + DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2" DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1" DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1" -DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1" +DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2" DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT" DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST" DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW" @@ -316,8 +318,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]: raise DatasetAdmissionError("dataset admission status is incompatible") storage = _object(document["storage"], "storage") if ( - set(storage) - != {"policy", "admitted", "canonical_root", "path_exposed"} + set(storage) != {"policy", "admitted", "canonical_root", "path_exposed"} or storage.get("policy") != "worker-d-only" or storage.get("admitted") is not True or storage.get("canonical_root") is not True @@ -338,8 +339,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]: "vendor_checksum_available", } or archive.get("filename") != "goose_3d_val.zip" - or archive.get("source_url") - != "https://goose-dataset.de/storage/goose_3d_val.zip" + or archive.get("source_url") != "https://goose-dataset.de/storage/goose_3d_val.zip" or archive.get("vendor_checksum_available") is not False ): raise DatasetAdmissionError("dataset archive admission is incompatible") @@ -385,9 +385,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]: ): raise DatasetAdmissionError("dataset preview identity is incompatible") point_count = _positive_integer(document.get("point_count"), "point_count") - source_point_count = _positive_integer( - document.get("source_point_count"), "source_point_count" - ) + source_point_count = _positive_integer(document.get("source_point_count"), "source_point_count") if point_count > 50_000 or point_count > source_point_count: raise DatasetAdmissionError("dataset preview point count is incompatible") points = document.get("points_xyz_m") @@ -456,20 +454,28 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]: point_count = _positive_integer(document.get("point_count"), "point_count") if point_count > 50_000: raise DatasetAdmissionError("dataset ground preview point count is incompatible") - for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"): + for key in ( + "current_ground", + "patchwork_ground", + "patchwork_assigned", + "ground_truth_ground", + "evaluated", + "current_disagreement", + "patchwork_disagreement", + ): values = document.get(key) if ( not isinstance(values, list) or len(values) != point_count or any( - not isinstance(value, int) - or isinstance(value, bool) - or value not in (0, 1) + not isinstance(value, int) or isinstance(value, bool) or value not in (0, 1) for value in values ) ): raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible") - metrics = _object(document.get("metrics"), "metrics") + metrics_by_provider = _object(document.get("metrics"), "metrics") + if set(metrics_by_provider) != {"current", "patchworkpp"}: + raise DatasetAdmissionError("dataset ground provider metrics are incompatible") expected_metrics = { "true_positive", "false_positive", @@ -484,40 +490,88 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]: "natural_ground_recall", "obstacle_non_ground_recall", } - if set(metrics) != expected_metrics: - raise DatasetAdmissionError("dataset ground metrics are incompatible") - for key, value in metrics.items(): - if key in {"true_positive", "false_positive", "false_negative", "true_negative"}: - _nonnegative_integer(value, f"metrics.{key}") - elif ( + for provider_id in ("current", "patchworkpp"): + metrics = _object(metrics_by_provider[provider_id], f"metrics.{provider_id}") + if set(metrics) != expected_metrics: + raise DatasetAdmissionError("dataset ground metrics are incompatible") + for key, value in metrics.items(): + if key in {"true_positive", "false_positive", "false_negative", "true_negative"}: + _nonnegative_integer(value, f"metrics.{provider_id}.{key}") + elif ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not np.isfinite(value) + or not 0 <= value <= 1 + ): + raise DatasetAdmissionError( + f"dataset ground metric {provider_id}.{key} is incompatible" + ) + latency = _object(document.get("latency_ms"), "latency_ms") + if set(latency) != {"current", "patchworkpp"}: + raise DatasetAdmissionError("dataset ground latency providers are incompatible") + for provider_id, value in latency.items(): + if ( not isinstance(value, (int, float)) or isinstance(value, bool) or not np.isfinite(value) - or not 0 <= value <= 1 + or value < 0 ): - raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible") - latency = document.get("latency_ms") + raise DatasetAdmissionError(f"dataset ground latency {provider_id} is incompatible") + providers = _object(document.get("providers"), "providers") + if set(providers) != {"current", "patchworkpp"}: + raise DatasetAdmissionError("dataset ground providers are incompatible") + current_provider = _object(providers["current"], "providers.current") if ( - not isinstance(latency, (int, float)) - or isinstance(latency, bool) - or not np.isfinite(latency) - or latency < 0 - ): - raise DatasetAdmissionError("dataset ground latency is incompatible") - provider = _object(document.get("provider"), "provider") - if ( - set(provider) != {"provider_id", "implementation_sha256", "ground_truth"} - or provider.get("provider_id") != "missioncore-local-percentile-ground/v1" - or provider.get("ground_truth") is not False + set(current_provider) != {"provider_id", "implementation_sha256", "ground_truth"} + or current_provider.get("provider_id") != "missioncore-local-percentile-ground/v1" + or current_provider.get("ground_truth") is not False ): raise DatasetAdmissionError("dataset ground provider is incompatible") - digest = provider.get("implementation_sha256") + digest = current_provider.get("implementation_sha256") if ( not isinstance(digest, str) or len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest) ): raise DatasetAdmissionError("dataset ground provider digest is incompatible") + patchwork_provider = _object(providers["patchworkpp"], "providers.patchworkpp") + if ( + set(patchwork_provider) + != { + "provider_id", + "source_url", + "source_tag", + "source_commit", + "binding_version", + "binary_sha256", + "platform", + "machine", + "ground_truth", + } + or patchwork_provider.get("provider_id") != "patchworkpp/v1.4.1" + or patchwork_provider.get("source_tag") != "v1.4.1" + or patchwork_provider.get("source_commit") != "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c" + or patchwork_provider.get("ground_truth") is not False + ): + raise DatasetAdmissionError("dataset Patchwork++ provider is incompatible") + binary_digest = patchwork_provider.get("binary_sha256") + if ( + not isinstance(binary_digest, str) + or len(binary_digest) != 64 + or any(character not in "0123456789abcdef" for character in binary_digest) + ): + raise DatasetAdmissionError("dataset Patchwork++ binary digest is incompatible") + assigned_fraction = document.get("patchwork_assigned_fraction") + if ( + not isinstance(assigned_fraction, (int, float)) + or isinstance(assigned_fraction, bool) + or not np.isfinite(assigned_fraction) + or not 0 <= assigned_fraction <= 1 + ): + raise DatasetAdmissionError("dataset Patchwork++ assigned fraction is incompatible") + input_profile = _object(document.get("input_profile"), "input_profile") + if input_profile != DEFAULT_GOOSE_PATCHWORK_PROFILE.to_dict(): + raise DatasetAdmissionError("dataset Patchwork++ input profile is incompatible") safety = _object(document.get("safety"), "safety") if safety != { "qualification_only": True, diff --git a/src/k1link/datasets/goose_benchmark.py b/src/k1link/datasets/goose_benchmark.py index 99ebf64..6588c6f 100644 --- a/src/k1link/datasets/goose_benchmark.py +++ b/src/k1link/datasets/goose_benchmark.py @@ -18,14 +18,22 @@ from k1link.datasets.goose_admission import ( GooseAdmissionError, read_goose_label_mapping, ) +from k1link.datasets.goose_profile import ( + DEFAULT_GOOSE_PATCHWORK_PROFILE, + GoosePatchworkProfile, +) from k1link.ground_segmentation import ( DEFAULT_GROUND_BENCHMARK_PROFILE, GroundBenchmarkProfile, + GroundSegmenter, LocalPercentileGroundSegmenter, + PatchworkPPGroundSegmenter, ) GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1" GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1" +GOOSE_GROUND_AB_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-ab-benchmark/v1" +GOOSE_GROUND_AB_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2" MAX_PREVIEW_POINTS: Final = 50_000 @@ -135,8 +143,7 @@ def benchmark_goose_current_ground( "ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(), "evaluated": evaluated[indices].astype(np.uint8).tolist(), "disagreement": ( - evaluated[indices] - & (prediction.ground_mask[indices] != ground_truth[indices]) + evaluated[indices] & (prediction.ground_mask[indices] != ground_truth[indices]) ) .astype(np.uint8) .tolist(), @@ -179,6 +186,204 @@ def benchmark_goose_current_ground( return report +def benchmark_goose_patchwork_ground( + dataset_root: Path, + *, + current_profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, + patchwork_profile: GoosePatchworkProfile = DEFAULT_GOOSE_PATCHWORK_PROFILE, + patchwork: GroundSegmenter | None = None, + patchwork_module_name: str = "pypatchworkpp", + preview_points: int = MAX_PREVIEW_POINTS, +) -> dict[str, Any]: + """Score current and pinned Patchwork++ providers against one GOOSE frame.""" + + root = dataset_root.expanduser().absolute() + if not _is_worker_dataset_root(root): + raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root") + manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json") + if manifest["status"] != "frame-ready": + raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking") + archive_sha256 = str(manifest["archive"]["sha256"]) + frame_id = str(manifest["frame"]["frame_id"]) + install = root / "goose-3d/v2025-08-22/installs" / archive_sha256 + frame_root = install / "frames" / frame_id + frame = read_semantic_kitti_frame( + frame_root / "points.bin", + frame_root / "labels.label", + ) + labels = read_goose_label_mapping(install / "goose_label_mapping.csv") + categories = np.asarray( + [labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels], + dtype=np.uint8, + ) + evaluated = categories != 0 + ground_truth = (categories == 2) | (categories == 3) + xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype( + np.float32, + copy=False, + ) + current = LocalPercentileGroundSegmenter(current_profile) + candidate = ( + patchwork + if patchwork is not None + else PatchworkPPGroundSegmenter.load( + patchwork_profile, + module_name=patchwork_module_name, + ) + ) + current_result = current.segment(xyzi) + candidate_result = candidate.segment(xyzi) + for result, label in ( + (current_result, "current"), + (candidate_result, "Patchwork++"), + ): + if result.ground_mask.shape != (frame.point_count,) or result.assigned_mask.shape != ( + frame.point_count, + ): + raise GooseAdmissionError(f"{label} result is not point-aligned") + current_metrics = _ground_metrics( + current_result.ground_mask, + ground_truth, + evaluated, + categories, + ) + candidate_metrics = _ground_metrics( + candidate_result.ground_mask, + ground_truth, + evaluated, + categories, + ) + candidate_assigned_fraction = float(np.mean(candidate_result.assigned_mask)) + profile_document = patchwork_profile.to_dict() + current_profile_document = _goose_current_profile_document(current_profile) + identity_document = { + "source_id": GOOSE_SOURCE_ID, + "archive_sha256": archive_sha256, + "frame_id": frame_id, + "source_point_count": frame.point_count, + "input_profile": profile_document, + "current_profile": current_profile_document, + "providers": { + "current": dict(current.identity), + "patchworkpp": dict(candidate.identity), + }, + } + identity_sha256 = _canonical_sha256(identity_document) + output_root = install / "benchmarks" / f"ground-ab-{identity_sha256}" + output_root.mkdir(parents=True, exist_ok=True) + report_path = output_root / "report.json" + if report_path.is_file(): + try: + existing = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise GooseAdmissionError("existing GOOSE A/B report is invalid") from exc + if ( + not isinstance(existing, dict) + or existing.get("schema_version") != GOOSE_GROUND_AB_BENCHMARK_SCHEMA + or existing.get("identity_sha256") != identity_sha256 + or not (output_root / "prediction.npz").is_file() + or not (output_root / "preview.json").is_file() + ): + raise GooseAdmissionError("existing GOOSE A/B benchmark is incomplete") + return existing + + prediction_path = output_root / "prediction.npz" + _atomic_npz( + prediction_path, + current_ground=current_result.ground_mask.astype(np.uint8), + patchwork_ground=candidate_result.ground_mask.astype(np.uint8), + patchwork_assigned=candidate_result.assigned_mask.astype(np.uint8), + ground_truth_ground=ground_truth.astype(np.uint8), + evaluated=evaluated.astype(np.uint8), + ) + sample_count = min(frame.point_count, preview_points) + indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64) + current_disagreement = evaluated & (current_result.ground_mask != ground_truth) + candidate_disagreement = evaluated & (candidate_result.ground_mask != ground_truth) + preview = { + "schema_version": GOOSE_GROUND_AB_PREVIEW_SCHEMA, + "source_id": GOOSE_SOURCE_ID, + "frame_id": frame_id, + "sampling": "deterministic-even-index", + "point_count": sample_count, + "current_ground": current_result.ground_mask[indices].astype(np.uint8).tolist(), + "patchwork_ground": candidate_result.ground_mask[indices].astype(np.uint8).tolist(), + "patchwork_assigned": (candidate_result.assigned_mask[indices].astype(np.uint8).tolist()), + "ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(), + "evaluated": evaluated[indices].astype(np.uint8).tolist(), + "current_disagreement": (current_disagreement[indices].astype(np.uint8).tolist()), + "patchwork_disagreement": (candidate_disagreement[indices].astype(np.uint8).tolist()), + "metrics": { + "current": current_metrics, + "patchworkpp": candidate_metrics, + }, + "latency_ms": { + "current": current_result.latency_ms, + "patchworkpp": candidate_result.latency_ms, + }, + "patchwork_assigned_fraction": candidate_assigned_fraction, + "providers": { + "current": dict(current.identity), + "patchworkpp": dict(candidate.identity), + }, + "input_profile": profile_document, + "safety": { + "qualification_only": True, + "navigation_or_safety_accepted": False, + }, + } + preview_path = output_root / "preview.json" + _atomic_json(preview_path, preview) + winner = ( + "patchworkpp" + if candidate_metrics["ground_iou"] > current_metrics["ground_iou"] + else "current" + if current_metrics["ground_iou"] > candidate_metrics["ground_iou"] + else "tie" + ) + report = { + "schema_version": GOOSE_GROUND_AB_BENCHMARK_SCHEMA, + "identity_sha256": identity_sha256, + "identity": identity_document, + "ground_truth": { + "source": "GOOSE point-wise semantic labels", + "ground_categories": ["artificial_ground", "natural_ground"], + "void_excluded": True, + "evaluated_points": int(np.count_nonzero(evaluated)), + }, + "providers": { + "current": { + "metrics": current_metrics, + "latency_ms": current_result.latency_ms, + "assigned_fraction": 1.0, + }, + "patchworkpp": { + "metrics": candidate_metrics, + "latency_ms": candidate_result.latency_ms, + "assigned_fraction": candidate_assigned_fraction, + }, + }, + "artifacts": { + "prediction_sha256": _sha256_file(prediction_path), + "preview_sha256": _sha256_file(preview_path), + }, + "decision": { + "status": "one-frame-diagnostic", + "promoted": False, + "winner_by_ground_iou": winner, + "reason": "one public frame does not qualify a production ground provider", + "next_gate": "validation-split qualification with degradation profiles", + }, + "scope": { + "patchworkpp_input_admitted": True, + "normalized_scan_produced": False, + "complete_vehicle_transform_known": False, + }, + } + _atomic_json(report_path, report) + return report + + def _is_worker_dataset_root(root: Path) -> bool: normalized = str(root).replace("\\", "/").rstrip("/").lower() return normalized == "/mnt/d/ndc_missioncore/datasets" @@ -216,6 +421,45 @@ def _binary_metrics( } +def _ground_metrics( + prediction: np.ndarray[Any, Any], + ground_truth: np.ndarray[Any, Any], + evaluated: np.ndarray[Any, Any], + categories: np.ndarray[Any, Any], +) -> dict[str, float | int]: + metrics = _binary_metrics(prediction, ground_truth, evaluated) + metrics.update( + { + "artificial_ground_recall": _recall(prediction, categories == 2), + "natural_ground_recall": _recall(prediction, categories == 3), + "obstacle_non_ground_recall": _recall(~prediction, categories == 4), + } + ) + return metrics + + +def _goose_current_profile_document( + profile: GroundBenchmarkProfile, +) -> dict[str, object]: + parameters = profile.to_dict()["current_baseline"] + return { + "schema_version": "missioncore.goose-current-ground-profile/v1", + "profile_id": "goose-native-local-percentile/v1", + "source_id": GOOSE_SOURCE_ID, + "representation": "native-scan", + "sensor_frame": "sensor/lidar/vls128_roof", + "provider": parameters, + "scope": { + "complete_vehicle_transform_required": False, + "normalized_scan_produced": False, + }, + "authority": { + "qualification_only": True, + "navigation_or_safety_accepted": False, + }, + } + + def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float: return _ratio( int(np.count_nonzero(prediction & truth)), @@ -259,9 +503,7 @@ def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None: def _atomic_json(path: Path, value: dict[str, Any]) -> None: if path.exists(): raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists") - encoded = ( - json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" - ) + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary: temporary_path = Path(temporary.name) temporary.write(encoded) diff --git a/src/k1link/datasets/goose_profile.py b/src/k1link/datasets/goose_profile.py new file mode 100644 index 0000000..e7517de --- /dev/null +++ b/src/k1link/datasets/goose_profile.py @@ -0,0 +1,117 @@ +"""Published, algorithm-scoped GOOSE VLS-128 ground profile.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Final + +GOOSE_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.goose-patchwork-profile/v1" +GOOSE_PAPER_URL: Final = "https://arxiv.org/pdf/2310.16788" +GOOSE_PAPER_SHA256: Final = "cb17f38c3ae498917966a63bc0e034cc5dd3fa4ad35873b121592f99026cc77e" +GOOSE_TF_DOT_URL: Final = "https://goose-dataset.de/docs/resources/mucar3_tf/mucar3_tf.dot" +GOOSE_TF_DOT_SHA256: Final = "1b10e905bcff17b90305ff02734b5e6bb7f5753d9b8e74cd7c43f3d0f32ef469" + + +@dataclass(frozen=True, slots=True) +class GoosePatchworkProfile: + """Narrow admission profile for one native GOOSE scan. + + Figure 3 of the official paper dimensions ``base_link``/INS at 0.64 m + above ground and the VLS-128 optical center 1.60 m above ``base_link``. + Their sum is the physical height Patchwork++ requires. This does not claim + a complete vehicle mounting transform and cannot produce a normalized scan. + """ + + profile_id: str = "goose-mucar3-vls128-patchwork/v1" + base_link_height_above_ground_m: float = 0.64 + lidar_height_above_base_link_m: float = 1.60 + patchwork_sensor_height_proxy_m: float = 2.24 + patchwork_minimum_range_m: float = 2.7 + patchwork_maximum_range_m: float = 80.0 + patchwork_height_evidence: str = "published-dimensioned-schematic" + + def __post_init__(self) -> None: + values = ( + self.base_link_height_above_ground_m, + self.lidar_height_above_base_link_m, + self.patchwork_sensor_height_proxy_m, + self.patchwork_minimum_range_m, + self.patchwork_maximum_range_m, + ) + if ( + not self.profile_id + or not all(math.isfinite(value) for value in values) + or self.base_link_height_above_ground_m <= 0 + or self.lidar_height_above_base_link_m <= 0 + or not math.isclose( + self.base_link_height_above_ground_m + self.lidar_height_above_base_link_m, + self.patchwork_sensor_height_proxy_m, + abs_tol=1e-9, + ) + or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m + or self.patchwork_height_evidence != "published-dimensioned-schematic" + ): + raise ValueError("GOOSE Patchwork++ profile is invalid") + + def to_dict(self) -> dict[str, object]: + return { + "schema_version": GOOSE_PATCHWORK_PROFILE_SCHEMA, + "profile_id": self.profile_id, + "source_id": "goose-3d/v2025-08-22", + "representation": "native-scan", + "sensor_frame": { + "frame_id": "sensor/lidar/vls128_roof", + "handedness": "right", + "x": "forward", + "y": "left", + "z": "up", + "one_revolution": True, + }, + "height": { + "base_link_above_ground_m": self.base_link_height_above_ground_m, + "lidar_above_base_link_m": self.lidar_height_above_base_link_m, + "sensor_above_ground_m": self.patchwork_sensor_height_proxy_m, + "evidence": self.patchwork_height_evidence, + }, + "patchworkpp": { + "provider_id": "patchworkpp/v1.4.1", + "minimum_range_m": self.patchwork_minimum_range_m, + "maximum_range_m": self.patchwork_maximum_range_m, + "enable_rnr": True, + "enable_rvpf": True, + "enable_tgr": True, + }, + "evidence": { + "dimensioned_schematic": { + "url": GOOSE_PAPER_URL, + "sha256": GOOSE_PAPER_SHA256, + "paper_version": "arxiv-2310.16788v2", + "figure": 3, + }, + "tf_topology": { + "url": GOOSE_TF_DOT_URL, + "sha256": GOOSE_TF_DOT_SHA256, + "numeric_transform_present": False, + }, + "first_frame_cross_check": { + "method": "independent-ground-near-field-plane", + "expected_ground_z_m": -self.patchwork_sensor_height_proxy_m, + "observed_intercept_range_m": [-2.18, -2.14], + "used_for_calibration": False, + }, + }, + "scope": { + "patchworkpp_eligible": True, + "normalized_scan_produced": False, + "complete_vehicle_transform_known": False, + "deskew_claimed": False, + }, + "authority": { + "qualification_only": True, + "navigation_or_safety_accepted": False, + }, + } + + +DEFAULT_GOOSE_PATCHWORK_PROFILE: Final = GoosePatchworkProfile() diff --git a/src/k1link/ground_segmentation.py b/src/k1link/ground_segmentation.py index 4c4d8d6..834b9ed 100644 --- a/src/k1link/ground_segmentation.py +++ b/src/k1link/ground_segmentation.py @@ -3,17 +3,23 @@ from __future__ import annotations import hashlib +import importlib import math +import platform import time from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from types import ModuleType from typing import Final, Protocol import numpy as np import numpy.typing as npt BoolArray = npt.NDArray[np.bool_] +PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus" +PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1" +PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c" class GroundSegmentationError(ValueError): @@ -142,6 +148,114 @@ class GroundSegmenter(Protocol): def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ... +class PatchworkGroundProfile(Protocol): + @property + def patchwork_sensor_height_proxy_m(self) -> float: ... + + @property + def patchwork_minimum_range_m(self) -> float: ... + + @property + def patchwork_maximum_range_m(self) -> float: ... + + +class PatchworkPPGroundSegmenter: + """Runtime-only adapter for the pinned official Patchwork++ Python binding.""" + + def __init__( + self, + module: ModuleType, + profile: PatchworkGroundProfile, + *, + source_commit: str = PATCHWORKPP_SOURCE_COMMIT, + source_tag: str = PATCHWORKPP_SOURCE_TAG, + ) -> None: + if len(source_commit) != 40 or any( + character not in "0123456789abcdef" for character in source_commit + ): + raise GroundSegmentationError("Patchwork++ source commit is invalid") + if source_tag != PATCHWORKPP_SOURCE_TAG: + raise GroundSegmentationError("Patchwork++ source tag is not admitted") + module_path_value = getattr(module, "__file__", None) + if not isinstance(module_path_value, str): + raise GroundSegmentationError("Patchwork++ module has no verifiable binary") + module_path = Path(module_path_value).resolve(strict=True) + params = module.Parameters() + params.sensor_height = profile.patchwork_sensor_height_proxy_m + params.min_range = profile.patchwork_minimum_range_m + params.max_range = profile.patchwork_maximum_range_m + params.enable_RNR = True + params.enable_RVPF = True + params.enable_TGR = True + params.verbose = False + self._estimator = module.patchworkpp(params) + self._identity = { + "provider_id": "patchworkpp/v1.4.1", + "source_url": PATCHWORKPP_SOURCE_URL, + "source_tag": source_tag, + "source_commit": source_commit, + "binding_version": str(getattr(module, "__version__", "unknown")), + "binary_sha256": _sha256(module_path), + "platform": platform.system().lower(), + "machine": platform.machine().lower(), + "ground_truth": False, + } + + @classmethod + def load( + cls, + profile: PatchworkGroundProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, + *, + module_name: str = "pypatchworkpp", + source_commit: str = PATCHWORKPP_SOURCE_COMMIT, + source_tag: str = PATCHWORKPP_SOURCE_TAG, + ) -> PatchworkPPGroundSegmenter: + try: + module = importlib.import_module(module_name) + except ImportError as exc: + raise GroundSegmentationError( + "Pinned Patchwork++ Python binding is unavailable" + ) from exc + return cls( + module, + profile, + source_commit=source_commit, + source_tag=source_tag, + ) + + @property + def identity(self) -> Mapping[str, object]: + return self._identity + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: + points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32) + started = time.perf_counter_ns() + self._estimator.estimateGround(points) + latency_ms = (time.perf_counter_ns() - started) / 1_000_000 + ground_indices = np.asarray( + self._estimator.getGroundIndices(), + dtype=np.int64, + ).reshape((-1,)) + nonground_indices = np.asarray( + self._estimator.getNongroundIndices(), + dtype=np.int64, + ).reshape((-1,)) + _indices(ground_indices, points.shape[0], "Patchwork++ ground") + _indices(nonground_indices, points.shape[0], "Patchwork++ non-ground") + if np.intersect1d(ground_indices, nonground_indices).size: + raise GroundSegmentationError("Patchwork++ assigned one point twice") + ground = np.zeros(points.shape[0], dtype=np.bool_) + assigned = np.zeros(points.shape[0], dtype=np.bool_) + ground[ground_indices] = True + assigned[ground_indices] = True + assigned[nonground_indices] = True + return GroundSegmentation( + ground_mask=ground, + assigned_mask=assigned, + latency_ms=latency_ms, + ) + + class LocalPercentileGroundSegmenter: """Full-frame diagnostic extension of the existing E19 local ground heuristic.""" @@ -167,8 +281,7 @@ class LocalPercentileGroundSegmenter: counts = np.bincount(inverse, minlength=unique_cells.shape[0]) offsets = np.concatenate(([0], np.cumsum(counts))) cell_lookup = { - (int(cell[0]), int(cell[1])): cell_index - for cell_index, cell in enumerate(unique_cells) + (int(cell[0]), int(cell[1])): cell_index for cell_index, cell in enumerate(unique_cells) } neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1 ground = np.zeros(points.shape[0], dtype=np.bool_) @@ -189,9 +302,7 @@ class LocalPercentileGroundSegmenter: if neighbor_cell_index is None: continue neighbor_slices.append( - order[ - offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1] - ] + order[offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]] ) local_indices = np.concatenate(neighbor_slices) local_xyz = xyz[local_indices] @@ -236,3 +347,10 @@ def _sha256(path: Path) -> str: for chunk in iter(lambda: source.read(1024**2), b""): digest.update(chunk) return digest.hexdigest() + + +def _indices(value: npt.NDArray[np.int64], count: int, label: str) -> None: + if value.ndim != 1 or np.any(value < 0) or np.any(value >= count): + raise GroundSegmentationError(f"{label} indices are invalid") + if np.unique(value).size != value.size: + raise GroundSegmentationError(f"{label} indices are duplicated") diff --git a/tests/test_dataset_gateway.py b/tests/test_dataset_gateway.py index 56c5d3d..ce64bff 100644 --- a/tests/test_dataset_gateway.py +++ b/tests/test_dataset_gateway.py @@ -21,7 +21,11 @@ from k1link.datasets import ( read_semantic_kitti_frame, ) from k1link.datasets.goose_admission import admit_goose_validation -from k1link.datasets.goose_benchmark import benchmark_goose_current_ground +from k1link.datasets.goose_benchmark import ( + benchmark_goose_current_ground, + benchmark_goose_patchwork_ground, +) +from k1link.ground_segmentation import GroundSegmentation from k1link.web.lidar_api import build_lidar_router @@ -354,12 +358,49 @@ def test_goose_current_ground_is_scored_against_independent_labels( assert report["ground_truth"]["evaluated_points"] == 4 assert 0 <= report["metrics"]["ground_iou"] <= 1 assert report["decision"]["promoted"] is False - output = ( - install - / "benchmarks" - / f"current-ground-{report['identity_sha256']}" - / "preview.json" - ) - preview = read_dataset_ground_preview(output) + output = install / "benchmarks" / f"current-ground-{report['identity_sha256']}" / "preview.json" + preview = json.loads(output.read_text(encoding="utf-8")) assert preview["point_count"] == 4 assert len(preview["disagreement"]) == 4 + + class FakePatchwork: + identity = { + "provider_id": "patchworkpp/v1.4.1", + "source_url": "https://github.com/url-kaist/patchwork-plusplus", + "source_tag": "v1.4.1", + "source_commit": "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c", + "binding_version": "test", + "binary_sha256": "c" * 64, + "platform": "linux", + "machine": "x86_64", + "ground_truth": False, + } + + def segment(self, xyzi: np.ndarray) -> GroundSegmentation: + return GroundSegmentation( + ground_mask=np.asarray([True, True, False, False]), + assigned_mask=np.ones(xyzi.shape[0], dtype=np.bool_), + latency_ms=0.2, + ) + + ab_report = benchmark_goose_patchwork_ground( + root, + patchwork=FakePatchwork(), + preview_points=4, + ) + + assert ab_report["scope"]["patchworkpp_input_admitted"] is True + assert ab_report["scope"]["normalized_scan_produced"] is False + assert ab_report["decision"]["promoted"] is False + ab_output = ( + install / "benchmarks" / f"ground-ab-{ab_report['identity_sha256']}" / "preview.json" + ) + ab_preview = read_dataset_ground_preview(ab_output) + assert ab_preview["schema_version"] == "missioncore.dataset-ground-comparison-preview/v2" + assert ab_preview["input_profile"]["height"]["sensor_above_ground_m"] == 2.24 + assert len(ab_preview["patchwork_disagreement"]) == 4 + + ab_preview["input_profile"]["height"]["sensor_above_ground_m"] = 2.25 + ab_output.write_text(json.dumps(ab_preview), encoding="utf-8") + with pytest.raises(DatasetAdmissionError, match="input profile"): + read_dataset_ground_preview(ab_output)