feat(lidar): qualify Patchwork++ on GOOSE

This commit is contained in:
DCCONSTRUCTIONS
2026-07-25 15:22:28 +03:00
parent 951b40c870
commit 60ba64004b
15 changed files with 1082 additions and 256 deletions
+12 -102
View File
@@ -1,18 +1,14 @@
from __future__ import annotations
import hashlib
import importlib
import json
import math
import os
import platform
import re
import shutil
import time
from collections.abc import Mapping
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
from typing import Any, Final
import numpy as np
@@ -25,9 +21,21 @@ from k1link.ground_segmentation import (
GroundSegmenter,
LocalPercentileGroundSegmenter,
)
from k1link.ground_segmentation import (
PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT,
)
from k1link.ground_segmentation import (
PATCHWORKPP_SOURCE_TAG as PATCHWORKPP_SOURCE_TAG,
)
from k1link.ground_segmentation import (
PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL,
)
from k1link.ground_segmentation import (
GroundSegmentationError as LidarGroundError,
)
from k1link.ground_segmentation import (
PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter,
)
from .lidar_contract import LidarContractError, sensor_frame_xyzi
from .lidar_replay import LidarReplayPackV2
@@ -42,107 +50,9 @@ LIDAR_GROUND_MANIFEST_NAME: Final = "manifest.json"
LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz"
MAX_GROUND_FRAME_POINTS: Final = 200_000
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
_ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$")
class PatchworkPPGroundSegmenter:
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
def __init__(
self,
module: ModuleType,
profile: GroundBenchmarkProfile,
*,
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
source_tag: str = PATCHWORKPP_SOURCE_TAG,
) -> None:
if _GIT_SHA1.fullmatch(source_commit) is None:
raise LidarGroundError("Patchwork++ source commit is invalid")
if source_tag != PATCHWORKPP_SOURCE_TAG:
raise LidarGroundError("Patchwork++ source tag is not admitted")
module_path_value = getattr(module, "__file__", None)
if not isinstance(module_path_value, str):
raise LidarGroundError("Patchwork++ module has no verifiable binary")
module_path = Path(module_path_value).resolve(strict=True)
params = module.Parameters()
params.sensor_height = profile.patchwork_sensor_height_proxy_m
params.min_range = profile.patchwork_minimum_range_m
params.max_range = profile.patchwork_maximum_range_m
params.enable_RNR = True
params.enable_RVPF = True
params.enable_TGR = True
params.verbose = False
self._estimator = module.patchworkpp(params)
self._identity = {
"provider_id": "patchworkpp/v1.4.1",
"source_url": PATCHWORKPP_SOURCE_URL,
"source_tag": source_tag,
"source_commit": source_commit,
"binding_version": str(getattr(module, "__version__", "unknown")),
"binary_sha256": _sha256(module_path),
"platform": platform.system().lower(),
"machine": platform.machine().lower(),
"ground_truth": False,
}
@classmethod
def load(
cls,
profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
*,
module_name: str = "pypatchworkpp",
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
source_tag: str = PATCHWORKPP_SOURCE_TAG,
) -> PatchworkPPGroundSegmenter:
try:
module = importlib.import_module(module_name)
except ImportError as exc:
raise LidarGroundError("Pinned Patchwork++ Python binding is unavailable") from exc
return cls(
module,
profile,
source_commit=source_commit,
source_tag=source_tag,
)
@property
def identity(self) -> Mapping[str, object]:
return self._identity
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
started = time.perf_counter_ns()
self._estimator.estimateGround(points)
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
ground_indices = np.asarray(
self._estimator.getGroundIndices(),
dtype=np.int64,
).reshape((-1,))
nonground_indices = np.asarray(
self._estimator.getNongroundIndices(),
dtype=np.int64,
).reshape((-1,))
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
if np.intersect1d(ground_indices, nonground_indices).size:
raise LidarGroundError("Patchwork++ assigned one point twice")
ground = np.zeros(points.shape[0], dtype=np.bool_)
assigned = np.zeros(points.shape[0], dtype=np.bool_)
ground[ground_indices] = True
assigned[ground_indices] = True
assigned[nonground_indices] = True
return GroundSegmentation(
ground_mask=ground,
assigned_mask=assigned,
latency_ms=latency_ms,
)
class LidarGroundBenchmarkV1:
+14
View File
@@ -15,9 +15,17 @@ from k1link.datasets.gateway import (
read_semantic_kitti_frame,
)
from k1link.datasets.goose_benchmark import (
GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
GOOSE_GROUND_AB_PREVIEW_SCHEMA,
GOOSE_GROUND_BENCHMARK_SCHEMA,
GOOSE_GROUND_PREVIEW_SCHEMA,
benchmark_goose_current_ground,
benchmark_goose_patchwork_ground,
)
from k1link.datasets.goose_profile import (
DEFAULT_GOOSE_PATCHWORK_PROFILE,
GOOSE_PATCHWORK_PROFILE_SCHEMA,
GoosePatchworkProfile,
)
__all__ = [
@@ -25,9 +33,15 @@ __all__ = [
"DatasetAdmissionError",
"DatasetFrameError",
"DatasetPointFrame",
"DEFAULT_GOOSE_PATCHWORK_PROFILE",
"GOOSE_GROUND_AB_BENCHMARK_SCHEMA",
"GOOSE_GROUND_AB_PREVIEW_SCHEMA",
"GOOSE_GROUND_BENCHMARK_SCHEMA",
"GOOSE_GROUND_PREVIEW_SCHEMA",
"GOOSE_PATCHWORK_PROFILE_SCHEMA",
"GoosePatchworkProfile",
"benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground",
"configured_dataset_admission_manifest",
"configured_dataset_ground_preview",
"configured_dataset_preview",
+21 -1
View File
@@ -9,7 +9,10 @@ from typing import Annotated
import typer
from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
from k1link.datasets.goose_benchmark import (
benchmark_goose_current_ground,
benchmark_goose_patchwork_ground,
)
app = typer.Typer(
add_completion=False,
@@ -63,5 +66,22 @@ def benchmark_goose_current_ground_command(
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
@app.command("benchmark-goose-patchwork-ground")
def benchmark_goose_patchwork_ground_command(
dataset_root: Annotated[
Path,
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
],
) -> None:
"""Compare current and pinned Patchwork++ providers against GOOSE labels."""
try:
report = benchmark_goose_patchwork_ground(dataset_root)
except (GooseAdmissionError, ValueError) as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
if __name__ == "__main__":
app()
+88 -34
View File
@@ -16,10 +16,12 @@ from typing import Any, Final, Literal
import numpy as np
import numpy.typing as npt
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST"
DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW"
@@ -316,8 +318,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
raise DatasetAdmissionError("dataset admission status is incompatible")
storage = _object(document["storage"], "storage")
if (
set(storage)
!= {"policy", "admitted", "canonical_root", "path_exposed"}
set(storage) != {"policy", "admitted", "canonical_root", "path_exposed"}
or storage.get("policy") != "worker-d-only"
or storage.get("admitted") is not True
or storage.get("canonical_root") is not True
@@ -338,8 +339,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
"vendor_checksum_available",
}
or archive.get("filename") != "goose_3d_val.zip"
or archive.get("source_url")
!= "https://goose-dataset.de/storage/goose_3d_val.zip"
or archive.get("source_url") != "https://goose-dataset.de/storage/goose_3d_val.zip"
or archive.get("vendor_checksum_available") is not False
):
raise DatasetAdmissionError("dataset archive admission is incompatible")
@@ -385,9 +385,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
):
raise DatasetAdmissionError("dataset preview identity is incompatible")
point_count = _positive_integer(document.get("point_count"), "point_count")
source_point_count = _positive_integer(
document.get("source_point_count"), "source_point_count"
)
source_point_count = _positive_integer(document.get("source_point_count"), "source_point_count")
if point_count > 50_000 or point_count > source_point_count:
raise DatasetAdmissionError("dataset preview point count is incompatible")
points = document.get("points_xyz_m")
@@ -456,20 +454,28 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
point_count = _positive_integer(document.get("point_count"), "point_count")
if point_count > 50_000:
raise DatasetAdmissionError("dataset ground preview point count is incompatible")
for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"):
for key in (
"current_ground",
"patchwork_ground",
"patchwork_assigned",
"ground_truth_ground",
"evaluated",
"current_disagreement",
"patchwork_disagreement",
):
values = document.get(key)
if (
not isinstance(values, list)
or len(values) != point_count
or any(
not isinstance(value, int)
or isinstance(value, bool)
or value not in (0, 1)
not isinstance(value, int) or isinstance(value, bool) or value not in (0, 1)
for value in values
)
):
raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible")
metrics = _object(document.get("metrics"), "metrics")
metrics_by_provider = _object(document.get("metrics"), "metrics")
if set(metrics_by_provider) != {"current", "patchworkpp"}:
raise DatasetAdmissionError("dataset ground provider metrics are incompatible")
expected_metrics = {
"true_positive",
"false_positive",
@@ -484,40 +490,88 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
"natural_ground_recall",
"obstacle_non_ground_recall",
}
if set(metrics) != expected_metrics:
raise DatasetAdmissionError("dataset ground metrics are incompatible")
for key, value in metrics.items():
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
_nonnegative_integer(value, f"metrics.{key}")
elif (
for provider_id in ("current", "patchworkpp"):
metrics = _object(metrics_by_provider[provider_id], f"metrics.{provider_id}")
if set(metrics) != expected_metrics:
raise DatasetAdmissionError("dataset ground metrics are incompatible")
for key, value in metrics.items():
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
_nonnegative_integer(value, f"metrics.{provider_id}.{key}")
elif (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not np.isfinite(value)
or not 0 <= value <= 1
):
raise DatasetAdmissionError(
f"dataset ground metric {provider_id}.{key} is incompatible"
)
latency = _object(document.get("latency_ms"), "latency_ms")
if set(latency) != {"current", "patchworkpp"}:
raise DatasetAdmissionError("dataset ground latency providers are incompatible")
for provider_id, value in latency.items():
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not np.isfinite(value)
or not 0 <= value <= 1
or value < 0
):
raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible")
latency = document.get("latency_ms")
raise DatasetAdmissionError(f"dataset ground latency {provider_id} is incompatible")
providers = _object(document.get("providers"), "providers")
if set(providers) != {"current", "patchworkpp"}:
raise DatasetAdmissionError("dataset ground providers are incompatible")
current_provider = _object(providers["current"], "providers.current")
if (
not isinstance(latency, (int, float))
or isinstance(latency, bool)
or not np.isfinite(latency)
or latency < 0
):
raise DatasetAdmissionError("dataset ground latency is incompatible")
provider = _object(document.get("provider"), "provider")
if (
set(provider) != {"provider_id", "implementation_sha256", "ground_truth"}
or provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
or provider.get("ground_truth") is not False
set(current_provider) != {"provider_id", "implementation_sha256", "ground_truth"}
or current_provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
or current_provider.get("ground_truth") is not False
):
raise DatasetAdmissionError("dataset ground provider is incompatible")
digest = provider.get("implementation_sha256")
digest = current_provider.get("implementation_sha256")
if (
not isinstance(digest, str)
or len(digest) != 64
or any(character not in "0123456789abcdef" for character in digest)
):
raise DatasetAdmissionError("dataset ground provider digest is incompatible")
patchwork_provider = _object(providers["patchworkpp"], "providers.patchworkpp")
if (
set(patchwork_provider)
!= {
"provider_id",
"source_url",
"source_tag",
"source_commit",
"binding_version",
"binary_sha256",
"platform",
"machine",
"ground_truth",
}
or patchwork_provider.get("provider_id") != "patchworkpp/v1.4.1"
or patchwork_provider.get("source_tag") != "v1.4.1"
or patchwork_provider.get("source_commit") != "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
or patchwork_provider.get("ground_truth") is not False
):
raise DatasetAdmissionError("dataset Patchwork++ provider is incompatible")
binary_digest = patchwork_provider.get("binary_sha256")
if (
not isinstance(binary_digest, str)
or len(binary_digest) != 64
or any(character not in "0123456789abcdef" for character in binary_digest)
):
raise DatasetAdmissionError("dataset Patchwork++ binary digest is incompatible")
assigned_fraction = document.get("patchwork_assigned_fraction")
if (
not isinstance(assigned_fraction, (int, float))
or isinstance(assigned_fraction, bool)
or not np.isfinite(assigned_fraction)
or not 0 <= assigned_fraction <= 1
):
raise DatasetAdmissionError("dataset Patchwork++ assigned fraction is incompatible")
input_profile = _object(document.get("input_profile"), "input_profile")
if input_profile != DEFAULT_GOOSE_PATCHWORK_PROFILE.to_dict():
raise DatasetAdmissionError("dataset Patchwork++ input profile is incompatible")
safety = _object(document.get("safety"), "safety")
if safety != {
"qualification_only": True,
+247 -5
View File
@@ -18,14 +18,22 @@ from k1link.datasets.goose_admission import (
GooseAdmissionError,
read_goose_label_mapping,
)
from k1link.datasets.goose_profile import (
DEFAULT_GOOSE_PATCHWORK_PROFILE,
GoosePatchworkProfile,
)
from k1link.ground_segmentation import (
DEFAULT_GROUND_BENCHMARK_PROFILE,
GroundBenchmarkProfile,
GroundSegmenter,
LocalPercentileGroundSegmenter,
PatchworkPPGroundSegmenter,
)
GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1"
GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
GOOSE_GROUND_AB_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-ab-benchmark/v1"
GOOSE_GROUND_AB_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
MAX_PREVIEW_POINTS: Final = 50_000
@@ -135,8 +143,7 @@ def benchmark_goose_current_ground(
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
"disagreement": (
evaluated[indices]
& (prediction.ground_mask[indices] != ground_truth[indices])
evaluated[indices] & (prediction.ground_mask[indices] != ground_truth[indices])
)
.astype(np.uint8)
.tolist(),
@@ -179,6 +186,204 @@ def benchmark_goose_current_ground(
return report
def benchmark_goose_patchwork_ground(
dataset_root: Path,
*,
current_profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
patchwork_profile: GoosePatchworkProfile = DEFAULT_GOOSE_PATCHWORK_PROFILE,
patchwork: GroundSegmenter | None = None,
patchwork_module_name: str = "pypatchworkpp",
preview_points: int = MAX_PREVIEW_POINTS,
) -> dict[str, Any]:
"""Score current and pinned Patchwork++ providers against one GOOSE frame."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root")
manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json")
if manifest["status"] != "frame-ready":
raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking")
archive_sha256 = str(manifest["archive"]["sha256"])
frame_id = str(manifest["frame"]["frame_id"])
install = root / "goose-3d/v2025-08-22/installs" / archive_sha256
frame_root = install / "frames" / frame_id
frame = read_semantic_kitti_frame(
frame_root / "points.bin",
frame_root / "labels.label",
)
labels = read_goose_label_mapping(install / "goose_label_mapping.csv")
categories = np.asarray(
[labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels],
dtype=np.uint8,
)
evaluated = categories != 0
ground_truth = (categories == 2) | (categories == 3)
xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype(
np.float32,
copy=False,
)
current = LocalPercentileGroundSegmenter(current_profile)
candidate = (
patchwork
if patchwork is not None
else PatchworkPPGroundSegmenter.load(
patchwork_profile,
module_name=patchwork_module_name,
)
)
current_result = current.segment(xyzi)
candidate_result = candidate.segment(xyzi)
for result, label in (
(current_result, "current"),
(candidate_result, "Patchwork++"),
):
if result.ground_mask.shape != (frame.point_count,) or result.assigned_mask.shape != (
frame.point_count,
):
raise GooseAdmissionError(f"{label} result is not point-aligned")
current_metrics = _ground_metrics(
current_result.ground_mask,
ground_truth,
evaluated,
categories,
)
candidate_metrics = _ground_metrics(
candidate_result.ground_mask,
ground_truth,
evaluated,
categories,
)
candidate_assigned_fraction = float(np.mean(candidate_result.assigned_mask))
profile_document = patchwork_profile.to_dict()
current_profile_document = _goose_current_profile_document(current_profile)
identity_document = {
"source_id": GOOSE_SOURCE_ID,
"archive_sha256": archive_sha256,
"frame_id": frame_id,
"source_point_count": frame.point_count,
"input_profile": profile_document,
"current_profile": current_profile_document,
"providers": {
"current": dict(current.identity),
"patchworkpp": dict(candidate.identity),
},
}
identity_sha256 = _canonical_sha256(identity_document)
output_root = install / "benchmarks" / f"ground-ab-{identity_sha256}"
output_root.mkdir(parents=True, exist_ok=True)
report_path = output_root / "report.json"
if report_path.is_file():
try:
existing = json.loads(report_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise GooseAdmissionError("existing GOOSE A/B report is invalid") from exc
if (
not isinstance(existing, dict)
or existing.get("schema_version") != GOOSE_GROUND_AB_BENCHMARK_SCHEMA
or existing.get("identity_sha256") != identity_sha256
or not (output_root / "prediction.npz").is_file()
or not (output_root / "preview.json").is_file()
):
raise GooseAdmissionError("existing GOOSE A/B benchmark is incomplete")
return existing
prediction_path = output_root / "prediction.npz"
_atomic_npz(
prediction_path,
current_ground=current_result.ground_mask.astype(np.uint8),
patchwork_ground=candidate_result.ground_mask.astype(np.uint8),
patchwork_assigned=candidate_result.assigned_mask.astype(np.uint8),
ground_truth_ground=ground_truth.astype(np.uint8),
evaluated=evaluated.astype(np.uint8),
)
sample_count = min(frame.point_count, preview_points)
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
current_disagreement = evaluated & (current_result.ground_mask != ground_truth)
candidate_disagreement = evaluated & (candidate_result.ground_mask != ground_truth)
preview = {
"schema_version": GOOSE_GROUND_AB_PREVIEW_SCHEMA,
"source_id": GOOSE_SOURCE_ID,
"frame_id": frame_id,
"sampling": "deterministic-even-index",
"point_count": sample_count,
"current_ground": current_result.ground_mask[indices].astype(np.uint8).tolist(),
"patchwork_ground": candidate_result.ground_mask[indices].astype(np.uint8).tolist(),
"patchwork_assigned": (candidate_result.assigned_mask[indices].astype(np.uint8).tolist()),
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
"current_disagreement": (current_disagreement[indices].astype(np.uint8).tolist()),
"patchwork_disagreement": (candidate_disagreement[indices].astype(np.uint8).tolist()),
"metrics": {
"current": current_metrics,
"patchworkpp": candidate_metrics,
},
"latency_ms": {
"current": current_result.latency_ms,
"patchworkpp": candidate_result.latency_ms,
},
"patchwork_assigned_fraction": candidate_assigned_fraction,
"providers": {
"current": dict(current.identity),
"patchworkpp": dict(candidate.identity),
},
"input_profile": profile_document,
"safety": {
"qualification_only": True,
"navigation_or_safety_accepted": False,
},
}
preview_path = output_root / "preview.json"
_atomic_json(preview_path, preview)
winner = (
"patchworkpp"
if candidate_metrics["ground_iou"] > current_metrics["ground_iou"]
else "current"
if current_metrics["ground_iou"] > candidate_metrics["ground_iou"]
else "tie"
)
report = {
"schema_version": GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
"identity_sha256": identity_sha256,
"identity": identity_document,
"ground_truth": {
"source": "GOOSE point-wise semantic labels",
"ground_categories": ["artificial_ground", "natural_ground"],
"void_excluded": True,
"evaluated_points": int(np.count_nonzero(evaluated)),
},
"providers": {
"current": {
"metrics": current_metrics,
"latency_ms": current_result.latency_ms,
"assigned_fraction": 1.0,
},
"patchworkpp": {
"metrics": candidate_metrics,
"latency_ms": candidate_result.latency_ms,
"assigned_fraction": candidate_assigned_fraction,
},
},
"artifacts": {
"prediction_sha256": _sha256_file(prediction_path),
"preview_sha256": _sha256_file(preview_path),
},
"decision": {
"status": "one-frame-diagnostic",
"promoted": False,
"winner_by_ground_iou": winner,
"reason": "one public frame does not qualify a production ground provider",
"next_gate": "validation-split qualification with degradation profiles",
},
"scope": {
"patchworkpp_input_admitted": True,
"normalized_scan_produced": False,
"complete_vehicle_transform_known": False,
},
}
_atomic_json(report_path, report)
return report
def _is_worker_dataset_root(root: Path) -> bool:
normalized = str(root).replace("\\", "/").rstrip("/").lower()
return normalized == "/mnt/d/ndc_missioncore/datasets"
@@ -216,6 +421,45 @@ def _binary_metrics(
}
def _ground_metrics(
prediction: np.ndarray[Any, Any],
ground_truth: np.ndarray[Any, Any],
evaluated: np.ndarray[Any, Any],
categories: np.ndarray[Any, Any],
) -> dict[str, float | int]:
metrics = _binary_metrics(prediction, ground_truth, evaluated)
metrics.update(
{
"artificial_ground_recall": _recall(prediction, categories == 2),
"natural_ground_recall": _recall(prediction, categories == 3),
"obstacle_non_ground_recall": _recall(~prediction, categories == 4),
}
)
return metrics
def _goose_current_profile_document(
profile: GroundBenchmarkProfile,
) -> dict[str, object]:
parameters = profile.to_dict()["current_baseline"]
return {
"schema_version": "missioncore.goose-current-ground-profile/v1",
"profile_id": "goose-native-local-percentile/v1",
"source_id": GOOSE_SOURCE_ID,
"representation": "native-scan",
"sensor_frame": "sensor/lidar/vls128_roof",
"provider": parameters,
"scope": {
"complete_vehicle_transform_required": False,
"normalized_scan_produced": False,
},
"authority": {
"qualification_only": True,
"navigation_or_safety_accepted": False,
},
}
def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float:
return _ratio(
int(np.count_nonzero(prediction & truth)),
@@ -259,9 +503,7 @@ def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None:
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
if path.exists():
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
encoded = (
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
)
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
temporary_path = Path(temporary.name)
temporary.write(encoded)
+117
View File
@@ -0,0 +1,117 @@
"""Published, algorithm-scoped GOOSE VLS-128 ground profile."""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Final
GOOSE_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.goose-patchwork-profile/v1"
GOOSE_PAPER_URL: Final = "https://arxiv.org/pdf/2310.16788"
GOOSE_PAPER_SHA256: Final = "cb17f38c3ae498917966a63bc0e034cc5dd3fa4ad35873b121592f99026cc77e"
GOOSE_TF_DOT_URL: Final = "https://goose-dataset.de/docs/resources/mucar3_tf/mucar3_tf.dot"
GOOSE_TF_DOT_SHA256: Final = "1b10e905bcff17b90305ff02734b5e6bb7f5753d9b8e74cd7c43f3d0f32ef469"
@dataclass(frozen=True, slots=True)
class GoosePatchworkProfile:
"""Narrow admission profile for one native GOOSE scan.
Figure 3 of the official paper dimensions ``base_link``/INS at 0.64 m
above ground and the VLS-128 optical center 1.60 m above ``base_link``.
Their sum is the physical height Patchwork++ requires. This does not claim
a complete vehicle mounting transform and cannot produce a normalized scan.
"""
profile_id: str = "goose-mucar3-vls128-patchwork/v1"
base_link_height_above_ground_m: float = 0.64
lidar_height_above_base_link_m: float = 1.60
patchwork_sensor_height_proxy_m: float = 2.24
patchwork_minimum_range_m: float = 2.7
patchwork_maximum_range_m: float = 80.0
patchwork_height_evidence: str = "published-dimensioned-schematic"
def __post_init__(self) -> None:
values = (
self.base_link_height_above_ground_m,
self.lidar_height_above_base_link_m,
self.patchwork_sensor_height_proxy_m,
self.patchwork_minimum_range_m,
self.patchwork_maximum_range_m,
)
if (
not self.profile_id
or not all(math.isfinite(value) for value in values)
or self.base_link_height_above_ground_m <= 0
or self.lidar_height_above_base_link_m <= 0
or not math.isclose(
self.base_link_height_above_ground_m + self.lidar_height_above_base_link_m,
self.patchwork_sensor_height_proxy_m,
abs_tol=1e-9,
)
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
or self.patchwork_height_evidence != "published-dimensioned-schematic"
):
raise ValueError("GOOSE Patchwork++ profile is invalid")
def to_dict(self) -> dict[str, object]:
return {
"schema_version": GOOSE_PATCHWORK_PROFILE_SCHEMA,
"profile_id": self.profile_id,
"source_id": "goose-3d/v2025-08-22",
"representation": "native-scan",
"sensor_frame": {
"frame_id": "sensor/lidar/vls128_roof",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"one_revolution": True,
},
"height": {
"base_link_above_ground_m": self.base_link_height_above_ground_m,
"lidar_above_base_link_m": self.lidar_height_above_base_link_m,
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
"evidence": self.patchwork_height_evidence,
},
"patchworkpp": {
"provider_id": "patchworkpp/v1.4.1",
"minimum_range_m": self.patchwork_minimum_range_m,
"maximum_range_m": self.patchwork_maximum_range_m,
"enable_rnr": True,
"enable_rvpf": True,
"enable_tgr": True,
},
"evidence": {
"dimensioned_schematic": {
"url": GOOSE_PAPER_URL,
"sha256": GOOSE_PAPER_SHA256,
"paper_version": "arxiv-2310.16788v2",
"figure": 3,
},
"tf_topology": {
"url": GOOSE_TF_DOT_URL,
"sha256": GOOSE_TF_DOT_SHA256,
"numeric_transform_present": False,
},
"first_frame_cross_check": {
"method": "independent-ground-near-field-plane",
"expected_ground_z_m": -self.patchwork_sensor_height_proxy_m,
"observed_intercept_range_m": [-2.18, -2.14],
"used_for_calibration": False,
},
},
"scope": {
"patchworkpp_eligible": True,
"normalized_scan_produced": False,
"complete_vehicle_transform_known": False,
"deskew_claimed": False,
},
"authority": {
"qualification_only": True,
"navigation_or_safety_accepted": False,
},
}
DEFAULT_GOOSE_PATCHWORK_PROFILE: Final = GoosePatchworkProfile()
+123 -5
View File
@@ -3,17 +3,23 @@
from __future__ import annotations
import hashlib
import importlib
import math
import platform
import time
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from types import ModuleType
from typing import Final, Protocol
import numpy as np
import numpy.typing as npt
BoolArray = npt.NDArray[np.bool_]
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
class GroundSegmentationError(ValueError):
@@ -142,6 +148,114 @@ class GroundSegmenter(Protocol):
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ...
class PatchworkGroundProfile(Protocol):
@property
def patchwork_sensor_height_proxy_m(self) -> float: ...
@property
def patchwork_minimum_range_m(self) -> float: ...
@property
def patchwork_maximum_range_m(self) -> float: ...
class PatchworkPPGroundSegmenter:
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
def __init__(
self,
module: ModuleType,
profile: PatchworkGroundProfile,
*,
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
source_tag: str = PATCHWORKPP_SOURCE_TAG,
) -> None:
if len(source_commit) != 40 or any(
character not in "0123456789abcdef" for character in source_commit
):
raise GroundSegmentationError("Patchwork++ source commit is invalid")
if source_tag != PATCHWORKPP_SOURCE_TAG:
raise GroundSegmentationError("Patchwork++ source tag is not admitted")
module_path_value = getattr(module, "__file__", None)
if not isinstance(module_path_value, str):
raise GroundSegmentationError("Patchwork++ module has no verifiable binary")
module_path = Path(module_path_value).resolve(strict=True)
params = module.Parameters()
params.sensor_height = profile.patchwork_sensor_height_proxy_m
params.min_range = profile.patchwork_minimum_range_m
params.max_range = profile.patchwork_maximum_range_m
params.enable_RNR = True
params.enable_RVPF = True
params.enable_TGR = True
params.verbose = False
self._estimator = module.patchworkpp(params)
self._identity = {
"provider_id": "patchworkpp/v1.4.1",
"source_url": PATCHWORKPP_SOURCE_URL,
"source_tag": source_tag,
"source_commit": source_commit,
"binding_version": str(getattr(module, "__version__", "unknown")),
"binary_sha256": _sha256(module_path),
"platform": platform.system().lower(),
"machine": platform.machine().lower(),
"ground_truth": False,
}
@classmethod
def load(
cls,
profile: PatchworkGroundProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
*,
module_name: str = "pypatchworkpp",
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
source_tag: str = PATCHWORKPP_SOURCE_TAG,
) -> PatchworkPPGroundSegmenter:
try:
module = importlib.import_module(module_name)
except ImportError as exc:
raise GroundSegmentationError(
"Pinned Patchwork++ Python binding is unavailable"
) from exc
return cls(
module,
profile,
source_commit=source_commit,
source_tag=source_tag,
)
@property
def identity(self) -> Mapping[str, object]:
return self._identity
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
started = time.perf_counter_ns()
self._estimator.estimateGround(points)
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
ground_indices = np.asarray(
self._estimator.getGroundIndices(),
dtype=np.int64,
).reshape((-1,))
nonground_indices = np.asarray(
self._estimator.getNongroundIndices(),
dtype=np.int64,
).reshape((-1,))
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
if np.intersect1d(ground_indices, nonground_indices).size:
raise GroundSegmentationError("Patchwork++ assigned one point twice")
ground = np.zeros(points.shape[0], dtype=np.bool_)
assigned = np.zeros(points.shape[0], dtype=np.bool_)
ground[ground_indices] = True
assigned[ground_indices] = True
assigned[nonground_indices] = True
return GroundSegmentation(
ground_mask=ground,
assigned_mask=assigned,
latency_ms=latency_ms,
)
class LocalPercentileGroundSegmenter:
"""Full-frame diagnostic extension of the existing E19 local ground heuristic."""
@@ -167,8 +281,7 @@ class LocalPercentileGroundSegmenter:
counts = np.bincount(inverse, minlength=unique_cells.shape[0])
offsets = np.concatenate(([0], np.cumsum(counts)))
cell_lookup = {
(int(cell[0]), int(cell[1])): cell_index
for cell_index, cell in enumerate(unique_cells)
(int(cell[0]), int(cell[1])): cell_index for cell_index, cell in enumerate(unique_cells)
}
neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1
ground = np.zeros(points.shape[0], dtype=np.bool_)
@@ -189,9 +302,7 @@ class LocalPercentileGroundSegmenter:
if neighbor_cell_index is None:
continue
neighbor_slices.append(
order[
offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]
]
order[offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]]
)
local_indices = np.concatenate(neighbor_slices)
local_xyz = xyz[local_indices]
@@ -236,3 +347,10 @@ def _sha256(path: Path) -> str:
for chunk in iter(lambda: source.read(1024**2), b""):
digest.update(chunk)
return digest.hexdigest()
def _indices(value: npt.NDArray[np.int64], count: int, label: str) -> None:
if value.ndim != 1 or np.any(value < 0) or np.any(value >= count):
raise GroundSegmentationError(f"{label} indices are invalid")
if np.unique(value).size != value.size:
raise GroundSegmentationError(f"{label} indices are duplicated")