From 75a3e669d9c8b5d76d2ac97d7ad23758887c5e2a Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sat, 25 Jul 2026 02:09:27 +0300 Subject: [PATCH] feat(lidar): add ground segmentation diagnostic benchmark --- .../src/core/lidar/replayQuality.ts | 212 ++++ .../control-station/src/styles/responsive.css | 9 + .../control-station/src/styles/workspaces.css | 70 ++ .../src/workspaces/LidarQualityWorkspace.tsx | 130 ++- .../test/lidarReplayQuality.test.mjs | 114 ++ docs/10_EXTERNAL_PERCEPTION_WORKER.md | 10 + docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md | 49 +- .../0020-patchworkpp-vendor-map-boundary.md | 69 ++ .../perception/run_lidar_ground_benchmark.py | 104 ++ src/k1link/compute/__init__.py | 36 + src/k1link/compute/lidar_ground.py | 1028 +++++++++++++++++ src/k1link/web/app.py | 9 +- src/k1link/web/lidar_api.py | 104 ++ tests/test_lidar_ground.py | 306 +++++ 14 files changed, 2235 insertions(+), 15 deletions(-) create mode 100644 docs/adr/0020-patchworkpp-vendor-map-boundary.md create mode 100644 experiments/perception/run_lidar_ground_benchmark.py create mode 100644 src/k1link/compute/lidar_ground.py create mode 100644 tests/test_lidar_ground.py diff --git a/apps/control-station/src/core/lidar/replayQuality.ts b/apps/control-station/src/core/lidar/replayQuality.ts index 3604ab7..620ea8d 100644 --- a/apps/control-station/src/core/lidar/replayQuality.ts +++ b/apps/control-station/src/core/lidar/replayQuality.ts @@ -59,6 +59,62 @@ export interface LidarReplayDetail { stages: LidarStageReadiness[]; } +export interface LidarGroundProviderSummary { + providerId: string; + sourceCommit: string | null; +} + +export interface LidarGroundBenchmark { + benchmarkId: string; + replayPackId: string; + sessionId: string; + status: "diagnostic-only"; + frames: number; + points: number; + inputDomain: { + accepted: false; + representation: string; + physicalSensorHeightKnown: boolean; + sensorScanGeometryKnown: boolean; + reason: string; + }; + labels: { + status: "missing-independent-review"; + metricsAvailable: false; + }; + current: { + provider: LidarGroundProviderSummary; + groundFraction: LidarDistribution; + assignedFraction: LidarDistribution; + latencyMs: LidarDistribution; + }; + candidate: { + provider: LidarGroundProviderSummary; + groundFraction: LidarDistribution; + assignedFraction: LidarDistribution; + latencyMs: LidarDistribution; + }; + comparison: { + algorithmGroundIou: LidarDistribution; + groundDisagreementFraction: LidarDistribution; + isAccuracyMetric: false; + }; + decision: { + status: "do-not-promote-on-current-vendor-map"; + productionPromotion: false; + reasons: string[]; + nextGate: string; + }; + createdAtUtc: string | null; +} + +export interface LidarGroundBenchmarkCatalog { + configured: boolean; + validTotal: number; + invalidTotal: number; + items: LidarGroundBenchmark[]; +} + export class LidarReplayContractError extends Error {} export class LidarReplayApiError extends Error { @@ -73,8 +129,10 @@ type LidarFetch = ( ) => Promise; const SAFE_PACK_ID = /^lidar-replay-pack-[a-f0-9]{64}$/; +const SAFE_GROUND_BENCHMARK_ID = /^ground-benchmark-[a-f0-9]{64}$/; const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$/; const SHA256 = /^[a-f0-9]{64}$/; +const GIT_SHA1 = /^[a-f0-9]{40}$/; function record(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { @@ -258,6 +316,139 @@ export function parseLidarReplayDetail(value: unknown): LidarReplayDetail { }; } +function groundProvider( + value: unknown, + label: string, +): LidarGroundProviderSummary { + const source = record(value, label); + return { + providerId: string(source.provider_id, `${label}.provider_id`, SAFE_ID), + sourceCommit: + source.source_commit === undefined || source.source_commit === null + ? null + : string(source.source_commit, `${label}.source_commit`, GIT_SHA1), + }; +} + +function groundBranch( + value: unknown, + label: string, +): LidarGroundBenchmark["current"] { + const source = record(value, label); + return { + provider: groundProvider(source.provider, `${label}.provider`), + groundFraction: distribution( + source.ground_fraction, + `${label}.ground_fraction`, + ), + assignedFraction: distribution( + source.assigned_fraction, + `${label}.assigned_fraction`, + ), + latencyMs: distribution(source.latency_ms, `${label}.latency_ms`), + }; +} + +function groundBenchmark(value: unknown): LidarGroundBenchmark { + const source = record(value, "LiDAR ground benchmark"); + if (source.status !== "diagnostic-only") { + throw new LidarReplayContractError("Ground benchmark status несовместим"); + } + const inputDomain = record(source.input_domain, "input_domain"); + const labels = record(source.labels, "labels"); + const comparison = record(source.comparison, "comparison"); + const decision = record(source.decision, "decision"); + if ( + inputDomain.accepted !== false + || labels.status !== "missing-independent-review" + || labels.metrics_available !== false + || comparison.is_accuracy_metric !== false + || decision.status !== "do-not-promote-on-current-vendor-map" + || decision.production_promotion !== false + ) { + throw new LidarReplayContractError( + "Ground benchmark завышает readiness или accuracy", + ); + } + return { + benchmarkId: string( + source.benchmark_id, + "benchmark_id", + SAFE_GROUND_BENCHMARK_ID, + ), + replayPackId: string(source.replay_pack_id, "replay_pack_id", SAFE_PACK_ID), + sessionId: string(source.session_id, "session_id", SAFE_ID), + status: "diagnostic-only", + frames: integer(source.frames, "frames"), + points: integer(source.points, "points"), + inputDomain: { + accepted: false, + representation: string( + inputDomain.representation, + "input_domain.representation", + SAFE_ID, + ), + physicalSensorHeightKnown: boolean( + inputDomain.physical_sensor_height_known, + "input_domain.physical_sensor_height_known", + ), + sensorScanGeometryKnown: boolean( + inputDomain.sensor_scan_geometry_known, + "input_domain.sensor_scan_geometry_known", + ), + reason: string(inputDomain.reason, "input_domain.reason"), + }, + labels: { + status: "missing-independent-review", + metricsAvailable: false, + }, + current: groundBranch(source.current, "current"), + candidate: groundBranch(source.candidate, "candidate"), + comparison: { + algorithmGroundIou: distribution( + comparison.algorithm_to_algorithm_ground_iou, + "comparison.algorithm_ground_iou", + ), + groundDisagreementFraction: distribution( + comparison.ground_disagreement_fraction, + "comparison.ground_disagreement_fraction", + ), + isAccuracyMetric: false, + }, + decision: { + status: "do-not-promote-on-current-vendor-map", + productionPromotion: false, + reasons: array(decision.reasons, "decision.reasons").map((reason) => + string(reason, "decision.reason") + ), + nextGate: string(decision.next_gate, "decision.next_gate"), + }, + createdAtUtc: + source.created_at_utc === null || source.created_at_utc === undefined + ? null + : string(source.created_at_utc, "created_at_utc"), + }; +} + +export function parseLidarGroundBenchmarkCatalog( + value: unknown, +): LidarGroundBenchmarkCatalog { + const source = record(value, "LiDAR ground catalog"); + if ( + source.schema_version + !== "missioncore.lidar-ground-benchmark-catalog/v1" + || source.access !== "read-only" + ) { + throw new LidarReplayContractError("LiDAR ground catalog несовместим"); + } + return { + configured: boolean(source.configured, "configured"), + validTotal: integer(source.valid_total, "valid_total"), + invalidTotal: integer(source.invalid_total, "invalid_total"), + items: array(source.items, "items").map(groundBenchmark), + }; +} + async function responseJson( response: Response, fallback: string, @@ -309,3 +500,24 @@ export async function fetchLidarReplayDetail( await responseJson(response, "Не удалось получить LiDAR quality report."), ); } + +export async function fetchLidarGroundBenchmarks( + packId: string, + options: { signal?: AbortSignal; fetcher?: LidarFetch } = {}, +): Promise { + if (!SAFE_PACK_ID.test(packId)) { + throw new LidarReplayContractError("Некорректный LiDAR pack id"); + } + const fetcher = options.fetcher ?? fetch; + const response = await fetcher( + `/api/v1/lidar/ground-benchmarks?pack_id=${encodeURIComponent(packId)}&limit=20`, + { + method: "GET", + headers: { Accept: "application/json" }, + signal: options.signal, + }, + ); + return parseLidarGroundBenchmarkCatalog( + await responseJson(response, "Не удалось получить LiDAR ground benchmark."), + ); +} diff --git a/apps/control-station/src/styles/responsive.css b/apps/control-station/src/styles/responsive.css index 8ebd791..ff34d0d 100644 --- a/apps/control-station/src/styles/responsive.css +++ b/apps/control-station/src/styles/responsive.css @@ -22,6 +22,10 @@ grid-template-columns: repeat(2, minmax(0, 1fr)); } + .lidar-ground-metrics { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .pipeline-strip { grid-template-columns: repeat(4, minmax(0, 1fr)); } @@ -162,6 +166,11 @@ grid-template-columns: 1fr; } + .lidar-ground-metrics, + .lidar-ground-gates { + grid-template-columns: 1fr; + } + .polygon-run-identity dl { grid-template-columns: 1fr; } diff --git a/apps/control-station/src/styles/workspaces.css b/apps/control-station/src/styles/workspaces.css index b5e1baf..eb927b2 100644 --- a/apps/control-station/src/styles/workspaces.css +++ b/apps/control-station/src/styles/workspaces.css @@ -954,6 +954,76 @@ margin-top: 0.8rem; } +.lidar-ground-benchmark { + min-width: 0; +} + +.lidar-ground-metrics { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.65rem; + margin-top: 1rem; +} + +.lidar-ground-metrics > div { + display: grid; + gap: 0.3rem; + border: 1px solid var(--station-hairline); + border-radius: 0.8rem; + background: rgb(255 255 255 / 0.025); + padding: 0.75rem; +} + +.lidar-ground-metrics span, +.lidar-ground-metrics small, +.lidar-ground-gates p, +.lidar-ground-footer, +.lidar-ground-empty { + color: var(--nodedc-text-muted); + font-size: 0.62rem; + line-height: 1.45; +} + +.lidar-ground-metrics strong { + color: var(--nodedc-text-primary); + font-size: 1.15rem; + font-weight: 650; +} + +.lidar-ground-gates { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 0.65rem; + margin-top: 0.75rem; +} + +.lidar-ground-gates > div { + display: grid; + align-content: start; + gap: 0.55rem; + border-left: 1px solid var(--station-hairline); + padding-left: 0.7rem; +} + +.lidar-ground-gates p, +.lidar-ground-empty { + margin: 0; +} + +.lidar-ground-footer { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + gap: 0.5rem; + margin-top: 0.85rem; + border-top: 1px solid var(--station-hairline); + padding-top: 0.7rem; +} + +.lidar-ground-empty { + margin-top: 1rem; +} + .overview-grid { display: grid; grid-template-columns: minmax(0, 1.55fr) minmax(19rem, 0.75fr); diff --git a/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx b/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx index 3e73a15..d23e35a 100644 --- a/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx +++ b/apps/control-station/src/workspaces/LidarQualityWorkspace.tsx @@ -6,8 +6,10 @@ import { } from "@nodedc/ui-react"; import { + fetchLidarGroundBenchmarks, fetchLidarReplayCatalog, fetchLidarReplayDetail, + type LidarGroundBenchmark, type LidarReplayCatalog, type LidarReplayDetail, type LidarStageReadiness, @@ -53,6 +55,8 @@ export function LidarQualityWorkspace({ }) { const [catalog, setCatalog] = useState(null); const [detail, setDetail] = useState(null); + const [groundBenchmark, setGroundBenchmark] = + useState(null); const [selectedPackId, setSelectedPackId] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -72,17 +76,25 @@ export function LidarQualityWorkspace({ const target = selectedPackId ?? nextCatalog.items[0]?.packId ?? null; if (!target) { setDetail(null); + setGroundBenchmark(null); return; } - const nextDetail = await fetchLidarReplayDetail(target, { - signal: controller.signal, - }); + const [nextDetail, groundCatalog] = await Promise.all([ + fetchLidarReplayDetail(target, { + signal: controller.signal, + }), + fetchLidarGroundBenchmarks(target, { + signal: controller.signal, + }), + ]); if (controller.signal.aborted) return; setSelectedPackId(target); setDetail(nextDetail); + setGroundBenchmark(groundCatalog.items[0] ?? null); } catch (loadError) { if (controller.signal.aborted) return; setDetail(null); + setGroundBenchmark(null); setError(errorMessage(loadError)); } finally { if (!controller.signal.aborted) setLoading(false); @@ -226,6 +238,118 @@ export function LidarQualityWorkspace({ + +
+
+ GROUND A/B · L2 +

Текущий heuristic против Patchwork++

+
+ + {groundBenchmark ? "Diagnostic only" : "Нет результата"} + +
+ {groundBenchmark ? ( + <> +
+
+ Текущий ground p50 + + {formatFraction( + groundBenchmark.current.groundFraction.p50, + )} + + + {formatNumber( + groundBenchmark.current.latencyMs.p95, + 2, + )}{" "} + мс p95 + +
+
+ Patchwork++ ground p50 + + {formatFraction( + groundBenchmark.candidate.groundFraction.p50, + )} + + + {formatNumber( + groundBenchmark.candidate.latencyMs.p95, + 2, + )}{" "} + мс p95 + +
+
+ Algorithm IoU p50 + + {formatFraction( + groundBenchmark.comparison.algorithmGroundIou.p50, + )} + + Не является accuracy +
+
+ Disagreement p50 + + {formatFraction( + groundBenchmark.comparison + .groundDisagreementFraction.p50, + )} + + + {groundBenchmark.frames.toLocaleString("ru-RU")} кадров + +
+
+
+
+ + Входной контракт не принят + +

+ Patchwork++ ожидает sensor-centric scan и физическую высоту + сенсора; текущий point feed является vendor-mapped increment. +

+
+
+ Разметка не принята +

+ IoU, curb recall, low-obstacle recall и reflection-noise + rejection появятся только после независимого human review. +

+
+
+ Не продвигать +

+ Следующий gate: human-reviewed annotation subset или + принятый raw sensor scan с физической высотой сенсора. +

+
+
+
+ + {groundBenchmark.current.provider.providerId} + + + {groundBenchmark.candidate.provider.providerId} + {groundBenchmark.candidate.provider.sourceCommit + ? ` · ${groundBenchmark.candidate.provider.sourceCommit.slice(0, 12)}` + : ""} + +
+ + ) : ( +

+ Для выбранного replay pack ещё нет проверенного ground benchmark. +

+ )} + +
diff --git a/apps/control-station/test/lidarReplayQuality.test.mjs b/apps/control-station/test/lidarReplayQuality.test.mjs index e58feaa..fe88bd8 100644 --- a/apps/control-station/test/lidarReplayQuality.test.mjs +++ b/apps/control-station/test/lidarReplayQuality.test.mjs @@ -6,8 +6,10 @@ import { createServer } from "vite"; let server; let parseLidarReplayCatalog; let parseLidarReplayDetail; +let parseLidarGroundBenchmarkCatalog; let fetchLidarReplayCatalog; let fetchLidarReplayDetail; +let fetchLidarGroundBenchmarks; let LidarReplayContractError; let workspaceById; @@ -20,8 +22,10 @@ before(async () => { ({ parseLidarReplayCatalog, parseLidarReplayDetail, + parseLidarGroundBenchmarkCatalog, fetchLidarReplayCatalog, fetchLidarReplayDetail, + fetchLidarGroundBenchmarks, LidarReplayContractError, } = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts")); ({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts")); @@ -130,6 +134,90 @@ function detail() { }; } +function groundCatalog(overrides = {}) { + const provider = (providerId, sourceCommit = undefined) => ({ + provider_id: providerId, + source_commit: sourceCommit, + }); + const branch = (providerValue, groundP50, latencyP95) => ({ + provider: providerValue, + ground_fraction: { + ...distribution(), + p50: groundP50, + }, + assigned_fraction: { + ...distribution(), + minimum: 1, + mean: 1, + p50: 1, + p95: 1, + maximum: 1, + }, + latency_ms: { + ...distribution(), + p95: latencyP95, + }, + }); + return { + schema_version: "missioncore.lidar-ground-benchmark-catalog/v1", + configured: true, + valid_total: 1, + invalid_total: 0, + access: "read-only", + items: [{ + benchmark_id: `ground-benchmark-${"c".repeat(64)}`, + replay_pack_id: packId, + session_id: "20260719T220917Z_viewer_live", + status: "diagnostic-only", + frames: 66, + points: 226963, + input_domain: { + accepted: false, + representation: "vendor-map-increment", + physical_sensor_height_known: false, + sensor_scan_geometry_known: false, + reason: "Patchwork++ expects sensor-centric scans.", + }, + labels: { + status: "missing-independent-review", + metrics_available: false, + }, + current: branch( + provider("missioncore-local-percentile-ground/v1"), + 0.183, + 10.7, + ), + candidate: branch( + provider( + "patchworkpp/v1.4.1", + "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c", + ), + 0.005, + 0.29, + ), + comparison: { + algorithm_to_algorithm_ground_iou: { + ...distribution(), + p50: 0.029, + }, + ground_disagreement_fraction: { + ...distribution(), + p50: 0.179, + }, + is_accuracy_metric: false, + }, + decision: { + status: "do-not-promote-on-current-vendor-map", + production_promotion: false, + reasons: ["candidate input domain is not accepted"], + next_gate: "human-reviewed annotation subset", + }, + created_at_utc: "2026-07-25T01:00:00Z", + }], + ...overrides, + }; +} + function jsonResponse(payload, status = 200) { return new Response(JSON.stringify(payload), { status, @@ -157,22 +245,48 @@ test("LiDAR contract refuses replay that did not pass equivalence", () => { ); }); +test("ground benchmark stays diagnostic until input and labels are accepted", () => { + const parsed = parseLidarGroundBenchmarkCatalog(groundCatalog()); + + assert.equal(parsed.items[0].status, "diagnostic-only"); + assert.equal(parsed.items[0].inputDomain.accepted, false); + assert.equal(parsed.items[0].labels.metricsAvailable, false); + assert.equal(parsed.items[0].candidate.groundFraction.p50, 0.005); + assert.equal(parsed.items[0].decision.productionPromotion, false); + + const promoted = groundCatalog(); + promoted.items[0].decision.production_promotion = true; + assert.throws( + () => parseLidarGroundBenchmarkCatalog(promoted), + LidarReplayContractError, + ); +}); + test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => { const calls = []; const fetcher = async (input, init) => { calls.push({ input: String(input), method: init?.method }); + if (String(input).includes("ground-benchmarks")) { + return jsonResponse(groundCatalog()); + } return String(input).includes(packId) ? jsonResponse(detail()) : jsonResponse(catalog()); }; const parsedCatalog = await fetchLidarReplayCatalog({ fetcher }); const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher }); + const ground = await fetchLidarGroundBenchmarks(packId, { fetcher }); assert.equal(parsedCatalog.validTotal, 1); assert.equal(parsedDetail.pack.packId, packId); + assert.equal(ground.validTotal, 1); assert.deepEqual(calls, [ { input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" }, { input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" }, + { + input: `/api/v1/lidar/ground-benchmarks?pack_id=${packId}&limit=20`, + method: "GET", + }, ]); assert.equal(workspaceById("lidar-quality").root, "data"); assert.equal(workspaceById("lidar-quality").kind, "lidar-quality"); diff --git a/docs/10_EXTERNAL_PERCEPTION_WORKER.md b/docs/10_EXTERNAL_PERCEPTION_WORKER.md index 3cb782f..ffae798 100644 --- a/docs/10_EXTERNAL_PERCEPTION_WORKER.md +++ b/docs/10_EXTERNAL_PERCEPTION_WORKER.md @@ -14,6 +14,16 @@ equivalence report. The React surface reads those reports through the read-only `/api/v1/lidar/replay-packs` boundary; CUDA/TensorRT and ROS do not move into the browser. +`missioncore.lidar-ground-benchmark/v1` extends the same boundary for L2. It +keeps point-aligned ground/assigned masks for the current local-percentile +proposal and a candidate provider, exact provider/source/binary identities, +host latency and algorithm disagreement. The first official Patchwork++ v1.4.1 +run is diagnostic-only: current K1 evidence is a vendor-map increment, not the +sensor-centric scan and physical-height contract Patchwork++ expects. The +read-only `/api/v1/lidar/ground-benchmarks` surface therefore publishes +`production_promotion=false` until an independent annotation generation or an +admitted raw scan closes the input gate. + ## Boundary ```text diff --git a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md index 9aaadd7..b256a7c 100644 --- a/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md +++ b/docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md @@ -1,7 +1,8 @@ # LiDAR worker: product value, evidence boundary and implementation roadmap Date: 2026-07-25 -Status: accepted architecture plan; L0 and L1 implemented +Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete, +independent labels pending Scope: real scanner records, replay and future live shadow processing Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering @@ -111,7 +112,7 @@ and training-domain fit. | Component | Correct use | Decision | | --- | --- | --- | -| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | First non-neural geometry baseline | +| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | v1.4.1 benchmarked; do not promote on the current vendor-map feed | | [Autoware CenterPoint](https://github.com/autowarefoundation/autoware_universe/tree/main/perception/autoware_lidar_centerpoint) | Mature ROS 2/TensorRT 3D detection and multi-frame reference | Second detector baseline after PointPillars | | [MMDetection3D](https://github.com/open-mmlab/mmdetection3d) | Training/evaluation harness and dataset adapters | Laboratory only | | [OpenPCDet](https://github.com/open-mmlab/OpenPCDet) | Alternative LiDAR detector benchmark/model zoo | Laboratory only; not the production runtime | @@ -192,17 +193,43 @@ Accepted real slice: The long interval tail is evidence to investigate in the scanner/transport quality line. It is not repaired or hidden by replay. -### L2 — geometric baseline +### L2 — geometric baseline — diagnostic A/B complete, accuracy gate open -- [ ] Run Patchwork++ over the frozen XYZI slice. -- [ ] Retain ground and non-ground outputs separately; never delete raw points. -- [ ] Label a small independent ground/obstacle evaluation set in CVAT or an - equivalent accepted annotation workspace. -- [ ] Measure ground IoU, curb/low-obstacle recall, reflection-noise rejection - and CPU/GPU latency. -- [ ] Compare against the current heuristic ground proposal branch. +- [x] Pin and run official Patchwork++ v1.4.1 at source commit + `3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`. +- [x] Compare it against a full-frame, point-aligned extension of the current + E19 local-percentile ground proposal. +- [x] Retain separate current/candidate ground and assigned masks for every + source point; raw replay remains unchanged. +- [x] Seal latency, ground-fraction, algorithm-IoU and disagreement + distributions in `missioncore.lidar-ground-benchmark/v1`. +- [x] Create an immutable eight-frame annotation template in which every point + starts as `ignore-unreviewed`; it is explicitly not ground truth. +- [ ] Complete independent human review for ground, curb, low obstacle, + reflection noise and other non-ground points. +- [ ] Measure accepted ground IoU, curb/low-obstacle recall and + reflection-noise rejection against that reviewed generation. -Exit: an evidence-backed decision to retain or reject Patchwork++. +The real diagnostic run is +`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`. +It covers all 66 frames and 226,963 points. The current local-percentile +proposal classified 18.31% ground at p50 with 9.74 ms p95 host latency. +Patchwork++ classified only 0.53% ground at p50 with 0.26 ms p95 host latency. +Their point-aligned ground IoU was 2.90% p50 and disagreement was 17.92% p50. +These last two values compare algorithms; they are not accuracy metrics. + +The result is an evidence-backed **do-not-promote** decision for the current K1 +feed. Patchwork++ is fast, but its input model assumes a sensor-centric scan +and physical sensor height. K1 `lio_pcl` is a vendor-mapped increment, its +physical sensor height is not encoded by the best-effort pose, and scan +geometry remains unknown. Translating the cloud until Patchwork++ looks +plausible would tune against the candidate and invalidate the comparison. + +The independent-label exit remains open. It can be closed by reviewing the +content-bound template +`ground-annotation-template-12e12eab0756c14f5e06adcaf13189e93188df9bb6d31224cb9b7d566ec15b18`, +or by acquiring an admitted raw sensor scan with known physical sensor height +and then producing a new benchmark generation. ### L3 — LiDAR-native 3D detection diff --git a/docs/adr/0020-patchworkpp-vendor-map-boundary.md b/docs/adr/0020-patchworkpp-vendor-map-boundary.md new file mode 100644 index 0000000..516a480 --- /dev/null +++ b/docs/adr/0020-patchworkpp-vendor-map-boundary.md @@ -0,0 +1,69 @@ +# ADR 0020: do not promote Patchwork++ on vendor-mapped LiDAR increments + +Date: 2026-07-25 +Status: accepted and implemented as a diagnostic gate + +## Context + +The L2 roadmap selected Patchwork++ as the first non-neural ground segmentation +baseline. The official implementation is designed around a sensor-centric +LiDAR scan, radial zones and a physical sensor height. Current K1 `lio_pcl` +evidence instead contains a vendor-mapped point increment in `map`, an +independent best-effort pose and no admitted raw sweep or scan geometry. The +pose describes vendor odometry; it does not prove the physical height of the +LiDAR above terrain. + +Treating a successful Patchwork++ call as a valid baseline would conflate API +compatibility with input-domain compatibility. Choosing a synthetic Z +translation until the output looks plausible would use the candidate itself to +define the normalization. + +## Decision + +1. Pin the official Patchwork++ v1.4.1 source at commit + `3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`. +2. Run it only behind the provider-neutral `lidar-ground/v1` diagnostic + boundary. +3. Bind the exact source commit and compiled binary SHA-256 into every result. +4. Preserve point-aligned ground and assigned masks separately from raw replay. +5. Compare it with the current local-percentile proposal using + algorithm-to-algorithm IoU, disagreement and latency. +6. Mark input-domain acceptance, labeled accuracy and production promotion + false for current K1 vendor-map evidence. +7. Never present algorithm IoU as ground IoU. +8. Create an all-ignore, content-bound annotation template; only a separate + human-reviewed generation may unlock ground IoU, curb/low-obstacle recall + and reflection-noise rejection. +9. Keep command, navigation and safety authority false. + +## Consequences + +- Patchwork++ remains a useful candidate for a future raw vehicle LiDAR feed. +- Its sub-millisecond host latency on the current slice does not compensate for + the unaccepted input domain. +- Current K1 work should prioritize independent labels and source evidence, not + threshold tuning around a mis-specified sensor model. +- The same immutable benchmark/API/React surface can compare a future raw scan, + replay or simulation provider without moving C++ processing into React. + +## Real diagnostic evidence + +Benchmark +`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984` +processed 66 frames and 226,963 points: + +| Measurement | Current local percentile | Patchwork++ | +| --- | ---: | ---: | +| Ground fraction p50 | 18.31% | 0.53% | +| Host latency p95 | 9.74 ms | 0.26 ms | + +Algorithm ground IoU was 2.90% p50 and point disagreement was 17.92% p50. +Neither is an accuracy metric. Independent labels are still missing. + +## References + +- `src/k1link/compute/lidar_ground.py` +- `experiments/perception/run_lidar_ground_benchmark.py` +- `src/k1link/web/lidar_api.py` +- `apps/control-station/src/workspaces/LidarQualityWorkspace.tsx` +- `docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md` diff --git a/experiments/perception/run_lidar_ground_benchmark.py b/experiments/perception/run_lidar_ground_benchmark.py new file mode 100644 index 0000000..5529247 --- /dev/null +++ b/experiments/perception/run_lidar_ground_benchmark.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute import ( + PATCHWORKPP_SOURCE_COMMIT, + PATCHWORKPP_SOURCE_TAG, + LidarGroundBenchmarkV1, + LidarReplayPackV2, + PatchworkPPGroundSegmenter, + build_lidar_ground_annotation_template, + build_lidar_ground_benchmark, +) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run diagnostic-only local-percentile vs Patchwork++ ground A/B " + "against one immutable LiDAR replay pack." + ) + ) + parser.add_argument("replay_pack", type=Path) + parser.add_argument("output_root", type=Path) + parser.add_argument( + "--annotation-root", + type=Path, + help="Optional root for an all-ignore human-review template.", + ) + parser.add_argument( + "--annotation-frames", + type=int, + default=8, + ) + parser.add_argument( + "--patchwork-module", + default="pypatchworkpp", + ) + parser.add_argument( + "--patchwork-source-tag", + default=PATCHWORKPP_SOURCE_TAG, + ) + parser.add_argument( + "--patchwork-source-commit", + default=PATCHWORKPP_SOURCE_COMMIT, + ) + return parser.parse_args() + + +def main() -> int: + arguments = _arguments() + replay = LidarReplayPackV2(arguments.replay_pack) + try: + patchwork = PatchworkPPGroundSegmenter.load( + module_name=arguments.patchwork_module, + source_tag=arguments.patchwork_source_tag, + source_commit=arguments.patchwork_source_commit, + ) + output = build_lidar_ground_benchmark( + replay, + arguments.output_root, + patchwork=patchwork, + ) + annotation = ( + build_lidar_ground_annotation_template( + replay, + arguments.annotation_root, + requested_frames=arguments.annotation_frames, + ) + if arguments.annotation_root is not None + else None + ) + finally: + replay.close() + + result = LidarGroundBenchmarkV1(output) + try: + print( + json.dumps( + { + "benchmark_id": result.benchmark_id, + "status": result.report["status"], + "decision": result.report["decision"], + "current": result.report["current"], + "candidate": result.report["candidate"], + "comparison": result.report["comparison"], + "annotation_template_id": ( + annotation.name if annotation is not None else None + ), + }, + ensure_ascii=False, + indent=2, + ) + ) + finally: + result.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/compute/__init__.py b/src/k1link/compute/__init__.py index 2de393d..83e957d 100644 --- a/src/k1link/compute/__init__.py +++ b/src/k1link/compute/__init__.py @@ -69,6 +69,25 @@ from .lidar_contract import ( lidar_readiness_document, sensor_frame_xyzi, ) +from .lidar_ground import ( + DEFAULT_GROUND_BENCHMARK_PROFILE, + LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA, + LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA, + LIDAR_GROUND_BENCHMARK_SCHEMA, + PATCHWORKPP_SOURCE_COMMIT, + PATCHWORKPP_SOURCE_TAG, + PATCHWORKPP_SOURCE_URL, + GroundBenchmarkProfile, + GroundSegmentation, + LidarGroundBenchmarkV1, + LidarGroundError, + LocalPercentileGroundSegmenter, + PatchworkPPGroundSegmenter, + build_lidar_ground_annotation_template, + build_lidar_ground_benchmark, + lidar_ground_benchmark_catalog_item, + score_ground_labels, +) from .lidar_replay import ( LIDAR_EQUIVALENCE_REPORT_SCHEMA, LIDAR_QUALITY_REPORT_SCHEMA, @@ -151,7 +170,12 @@ __all__ = [ "EVALUATION_PACK_SCHEMA", "EvaluationFrameRequest", "EvaluationPackFrame", + "GroundBenchmarkProfile", + "GroundSegmentation", "LatestWinsQueue", + "LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA", + "LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA", + "LIDAR_GROUND_BENCHMARK_SCHEMA", "LIDAR_EVIDENCE_PROFILE_SCHEMA", "LIDAR_EQUIVALENCE_REPORT_SCHEMA", "LIDAR_QUALITY_REPORT_SCHEMA", @@ -169,6 +193,8 @@ __all__ = [ "LidarPoseStatus", "LidarQualityMonitor", "LidarReadiness", + "LidarGroundBenchmarkV1", + "LidarGroundError", "LidarReplayError", "LidarReplayPackV2", "LidarReplayPointFrame", @@ -190,6 +216,12 @@ __all__ = [ "K1_LAB_LIDAR_PACK_V1_PROFILE", "K1_LIDAR_PACK_V2_PROFILE", "K1_LIVE_LIDAR_PROFILE", + "DEFAULT_GROUND_BENCHMARK_PROFILE", + "LocalPercentileGroundSegmenter", + "PATCHWORKPP_SOURCE_COMMIT", + "PATCHWORKPP_SOURCE_TAG", + "PATCHWORKPP_SOURCE_URL", + "PatchworkPPGroundSegmenter", "QueueSnapshot", "RecordedCalibratedFusion", "RecordedCalibratedFusionStore", @@ -224,6 +256,8 @@ __all__ = [ "prepare_recorded_qualification_slice", "assess_lidar_profile", "build_lidar_replay_pack_v2", + "build_lidar_ground_annotation_template", + "build_lidar_ground_benchmark", "DetectionFrame", "ObjectDetection", "RecordedPerceptionOverlayError", @@ -246,6 +280,8 @@ __all__ = [ "lidar_readiness_document", "lidar_pack_catalog_item", "lidar_pack_detail", + "lidar_ground_benchmark_catalog_item", + "score_ground_labels", "sensor_frame_xyzi", "verify_lidar_replay_equivalence", ] diff --git a/src/k1link/compute/lidar_ground.py b/src/k1link/compute/lidar_ground.py new file mode 100644 index 0000000..fc7febc --- /dev/null +++ b/src/k1link/compute/lidar_ground.py @@ -0,0 +1,1028 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import math +import os +import platform +import re +import shutil +import time +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from types import ModuleType +from typing import Any, Final, Protocol + +import numpy as np +import numpy.typing as npt + +from .lidar_contract import LidarContractError, sensor_frame_xyzi +from .lidar_replay import LidarReplayPackV2 + +LIDAR_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.lidar-ground-benchmark/v1" +LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA: Final = ( + "missioncore.lidar-ground-benchmark-report/v1" +) +LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA: Final = ( + "missioncore.lidar-ground-annotation-template/v1" +) +LIDAR_GROUND_RESULTS_NAME: Final = "ground-results.npz" +LIDAR_GROUND_REPORT_NAME: Final = "ground-report.json" +LIDAR_GROUND_MANIFEST_NAME: Final = "manifest.json" +LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz" + +PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus" +PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1" +PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c" + +_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$") +_ANNOTATION_TEMPLATE_ID = re.compile( + r"^ground-annotation-template-[a-f0-9]{64}$" +) +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$") + +BoolArray = npt.NDArray[np.bool_] +FloatArray = npt.NDArray[np.floating[Any]] + + +class LidarGroundError(ValueError): + """Ground benchmark evidence violates the diagnostic-only contract.""" + + +@dataclass(frozen=True, slots=True) +class GroundBenchmarkProfile: + profile_id: str = "k1-vendor-map-ground-ab/v1" + pose_binding_threshold_ms: float = 100.0 + current_cell_size_m: float = 0.5 + current_local_radius_m: float = 2.5 + current_lower_percentile: float = 8.0 + current_maximum_below_ground_m: float = 0.25 + current_maximum_above_ground_m: float = 0.12 + current_minimum_local_points: int = 8 + patchwork_sensor_height_proxy_m: float = 0.0 + patchwork_minimum_range_m: float = 0.1 + patchwork_maximum_range_m: float = 20.0 + + def to_dict(self) -> dict[str, object]: + return { + "schema_version": "missioncore.lidar-ground-benchmark-profile/v1", + "profile_id": self.profile_id, + "pose_binding": { + "basis": "nearest-recorded-host-monotonic-arrival", + "threshold_ms": self.pose_binding_threshold_ms, + }, + "current_baseline": { + "provider_id": "missioncore-local-percentile-ground/v1", + "derived_from": "local-ground-relative-object-support-v1", + "cell_size_m": self.current_cell_size_m, + "local_radius_m": self.current_local_radius_m, + "lower_percentile": self.current_lower_percentile, + "maximum_below_ground_m": self.current_maximum_below_ground_m, + "maximum_above_ground_m": self.current_maximum_above_ground_m, + "minimum_local_points": self.current_minimum_local_points, + }, + "candidate": { + "provider_id": "patchworkpp/v1.4.1", + "sensor_height_proxy_m": self.patchwork_sensor_height_proxy_m, + "minimum_range_m": self.patchwork_minimum_range_m, + "maximum_range_m": self.patchwork_maximum_range_m, + "enable_rnr": True, + "enable_rvpf": True, + "enable_tgr": True, + }, + "input_normalization": { + "current": "vendor-map-xyz", + "candidate": "best-effort-map-to-lidar-pose-inversion", + "physical_sensor_height_known": False, + "sensor_scan_geometry_known": False, + }, + "authority": { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + + +DEFAULT_GROUND_BENCHMARK_PROFILE: Final = GroundBenchmarkProfile() + + +@dataclass(frozen=True, slots=True) +class GroundSegmentation: + ground_mask: BoolArray + assigned_mask: BoolArray + latency_ms: float + + +class GroundSegmenter(Protocol): + @property + def identity(self) -> Mapping[str, object]: ... + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ... + + +class LocalPercentileGroundSegmenter: + """Full-frame diagnostic extension of the existing E19 local ground heuristic.""" + + def __init__(self, profile: GroundBenchmarkProfile) -> None: + self.profile = profile + + @property + def identity(self) -> Mapping[str, object]: + return { + "provider_id": "missioncore-local-percentile-ground/v1", + "implementation_sha256": _sha256(Path(__file__).resolve(strict=True)), + "ground_truth": False, + } + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: + points = _xyzi(xyzi) + started = time.perf_counter_ns() + xyz = points[:, :3].astype(np.float64, copy=False) + cell_size = self.profile.current_cell_size_m + cell_keys = np.floor(xyz[:, :2] / cell_size).astype(np.int64) + unique_cells, inverse = np.unique(cell_keys, axis=0, return_inverse=True) + ground = np.zeros(points.shape[0], dtype=np.bool_) + global_ground_z = float( + np.percentile(xyz[:, 2], self.profile.current_lower_percentile) + ) + radius_squared = self.profile.current_local_radius_m**2 + for cell_index in range(unique_cells.shape[0]): + point_indices = np.flatnonzero(inverse == cell_index) + if point_indices.size == 0: + continue + center_xy = np.median(xyz[point_indices, :2], axis=0) + delta_xy = xyz[:, :2] - center_xy + local = xyz[ + np.einsum("ij,ij->i", delta_xy, delta_xy) <= radius_squared, + 2, + ] + ground_z = ( + float( + np.percentile( + local, + self.profile.current_lower_percentile, + ) + ) + if local.size >= self.profile.current_minimum_local_points + else global_ground_z + ) + z = xyz[point_indices, 2] + ground[point_indices] = ( + (z >= ground_z - self.profile.current_maximum_below_ground_m) + & (z <= ground_z + self.profile.current_maximum_above_ground_m) + ) + latency_ms = (time.perf_counter_ns() - started) / 1_000_000 + return GroundSegmentation( + ground_mask=ground, + assigned_mask=np.ones(points.shape[0], dtype=np.bool_), + latency_ms=latency_ms, + ) + + +class PatchworkPPGroundSegmenter: + """Runtime-only adapter for the pinned official Patchwork++ Python binding.""" + + def __init__( + self, + module: ModuleType, + profile: GroundBenchmarkProfile, + *, + source_commit: str = PATCHWORKPP_SOURCE_COMMIT, + source_tag: str = PATCHWORKPP_SOURCE_TAG, + ) -> None: + if _GIT_SHA1.fullmatch(source_commit) is None: + raise LidarGroundError("Patchwork++ source commit is invalid") + if source_tag != PATCHWORKPP_SOURCE_TAG: + raise LidarGroundError("Patchwork++ source tag is not admitted") + module_path_value = getattr(module, "__file__", None) + if not isinstance(module_path_value, str): + raise LidarGroundError("Patchwork++ module has no verifiable binary") + module_path = Path(module_path_value).resolve(strict=True) + params = module.Parameters() + params.sensor_height = profile.patchwork_sensor_height_proxy_m + params.min_range = profile.patchwork_minimum_range_m + params.max_range = profile.patchwork_maximum_range_m + params.enable_RNR = True + params.enable_RVPF = True + params.enable_TGR = True + params.verbose = False + self._estimator = module.patchworkpp(params) + self._identity = { + "provider_id": "patchworkpp/v1.4.1", + "source_url": PATCHWORKPP_SOURCE_URL, + "source_tag": source_tag, + "source_commit": source_commit, + "binding_version": str(getattr(module, "__version__", "unknown")), + "binary_sha256": _sha256(module_path), + "platform": platform.system().lower(), + "machine": platform.machine().lower(), + "ground_truth": False, + } + + @classmethod + def load( + cls, + profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, + *, + module_name: str = "pypatchworkpp", + source_commit: str = PATCHWORKPP_SOURCE_COMMIT, + source_tag: str = PATCHWORKPP_SOURCE_TAG, + ) -> PatchworkPPGroundSegmenter: + try: + module = importlib.import_module(module_name) + except ImportError as exc: + raise LidarGroundError( + "Pinned Patchwork++ Python binding is unavailable" + ) from exc + return cls( + module, + profile, + source_commit=source_commit, + source_tag=source_tag, + ) + + @property + def identity(self) -> Mapping[str, object]: + return self._identity + + def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: + points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32) + started = time.perf_counter_ns() + self._estimator.estimateGround(points) + latency_ms = (time.perf_counter_ns() - started) / 1_000_000 + ground_indices = np.asarray( + self._estimator.getGroundIndices(), + dtype=np.int64, + ).reshape((-1,)) + nonground_indices = np.asarray( + self._estimator.getNongroundIndices(), + dtype=np.int64, + ).reshape((-1,)) + _indices(ground_indices, points.shape[0], "Patchwork++ ground") + _indices(nonground_indices, points.shape[0], "Patchwork++ non-ground") + if np.intersect1d(ground_indices, nonground_indices).size: + raise LidarGroundError("Patchwork++ assigned one point twice") + ground = np.zeros(points.shape[0], dtype=np.bool_) + assigned = np.zeros(points.shape[0], dtype=np.bool_) + ground[ground_indices] = True + assigned[ground_indices] = True + assigned[nonground_indices] = True + return GroundSegmentation( + ground_mask=ground, + assigned_mask=assigned, + latency_ms=latency_ms, + ) + + +class LidarGroundBenchmarkV1: + """Strict reader for immutable ground A/B diagnostic evidence.""" + + def __init__(self, root: Path) -> None: + candidate = root.expanduser().absolute() + if candidate.is_symlink(): + raise LidarGroundError("Ground benchmark directory cannot be a symlink") + self.root = candidate.resolve(strict=True) + if not self.root.is_dir() or _BENCHMARK_ID.fullmatch(self.root.name) is None: + raise LidarGroundError("Ground benchmark id is invalid") + self.manifest = _read_json(self.root / LIDAR_GROUND_MANIFEST_NAME) + self.identity = _object(self.manifest.get("identity"), "ground identity") + identity_sha256 = self.manifest.get("identity_sha256") + if ( + self.manifest.get("schema_version") != LIDAR_GROUND_BENCHMARK_SCHEMA + or self.identity.get("schema_version") != LIDAR_GROUND_BENCHMARK_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(self.identity)).hexdigest() + != identity_sha256 + or self.root.name != f"ground-benchmark-{identity_sha256}" + or self.manifest.get("benchmark_id") != self.root.name + ): + raise LidarGroundError("Ground benchmark identity is invalid") + artifacts = _validate_artifacts(self.root, self.manifest.get("artifacts")) + self.arrays = np.load(artifacts["ground-results"], allow_pickle=False) + self.report = _read_json(artifacts["ground-report"]) + try: + _validate_ground_arrays(self.arrays, self.identity) + input_domain = _object( + self.report.get("input_domain"), + "ground input domain", + ) + labels = _object(self.report.get("labels"), "ground labels") + decision = _object(self.report.get("decision"), "ground decision") + if ( + _ground_logical_sha256(self.arrays) + != self.identity.get("logical_results_sha256") + or self.report.get("schema_version") + != LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA + or self.report.get("benchmark_id") != self.root.name + or self.report.get("replay_pack_id") + != self.identity.get("replay_pack_id") + or self.report.get("status") != "diagnostic-only" + or input_domain.get("accepted") is not False + or labels.get("status") != "missing-independent-review" + or labels.get("metrics_available") is not False + or decision.get("status") + != "do-not-promote-on-current-vendor-map" + or decision.get("production_promotion") is not False + ): + raise LidarGroundError("Ground benchmark report is incompatible") + except BaseException: + self.close() + raise + self.benchmark_id = self.root.name + + def close(self) -> None: + self.arrays.close() + + +def build_lidar_ground_benchmark( + replay: LidarReplayPackV2, + output_root: Path, + *, + patchwork: GroundSegmenter, + profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE, +) -> Path: + """Run a diagnostic A/B without claiming labeled accuracy or safety.""" + + if replay.pose_frame_count < 1: + raise LidarGroundError("Ground A/B requires recorded pose evidence") + current = LocalPercentileGroundSegmenter(profile) + point_offsets = np.asarray(replay.arrays["point_offsets"], dtype=" profile.pose_binding_threshold_ms: + raise LidarGroundError("Ground A/B point frame has no admitted pose") + try: + candidate_input = sensor_frame_xyzi( + point.decoded_view(), + pose.decoded_view(), + ) + except LidarContractError as exc: + raise LidarGroundError("Ground A/B sensor conversion failed") from exc + current_input = np.empty((point.xyz_map.shape[0], 4), dtype=np.float32) + current_input[:, :3] = point.xyz_map.astype(np.float32) + current_input[:, 3] = point.intensity.astype(np.float32) / 255.0 + current_result = current.segment(current_input) + candidate_result = patchwork.segment(candidate_input) + _segmentation(current_result, point.xyz_map.shape[0], "current") + _segmentation(candidate_result, point.xyz_map.shape[0], "candidate") + start = int(point_offsets[frame_index]) + end = int(point_offsets[frame_index + 1]) + current_ground[start:end] = current_result.ground_mask.astype(np.uint8) + candidate_ground[start:end] = candidate_result.ground_mask.astype(np.uint8) + candidate_assigned[start:end] = candidate_result.assigned_mask.astype(np.uint8) + current_latency.append(current_result.latency_ms) + candidate_latency.append(candidate_result.latency_ms) + pose_delta_ms.append(delta_ms) + current_fraction.append(float(np.mean(current_result.ground_mask))) + candidate_fraction.append(float(np.mean(candidate_result.ground_mask))) + candidate_assigned_fraction.append( + float(np.mean(candidate_result.assigned_mask)) + ) + intersection = int( + np.count_nonzero( + current_result.ground_mask & candidate_result.ground_mask + ) + ) + union = int( + np.count_nonzero( + current_result.ground_mask | candidate_result.ground_mask + ) + ) + inter_provider_iou.append(float(intersection / union) if union else 1.0) + disagreement_fraction.append( + float( + np.mean( + current_result.ground_mask != candidate_result.ground_mask + ) + ) + ) + + arrays: dict[str, npt.NDArray[Any]] = { + "point_offsets": point_offsets, + "point_capture_sequence": np.asarray( + replay.arrays["point_capture_sequence"], + dtype=" Path: + """Create an all-ignore, immutable review template; never fabricate labels.""" + + if not 1 <= requested_frames <= 64: + raise LidarGroundError("Ground annotation frame count is invalid") + selected = np.unique( + np.linspace( + 0, + replay.point_frame_count - 1, + num=min(requested_frames, replay.point_frame_count), + dtype=np.int64, + ) + ) + source_offsets = np.asarray(replay.arrays["point_offsets"], dtype=np.int64) + selected_counts = np.asarray( + [ + int(source_offsets[index + 1] - source_offsets[index]) + for index in selected + ], + dtype=" dict[str, object]: + """Score reviewed point labels; label zero remains excluded, never negative.""" + + values = np.asarray(labels, dtype=np.uint8).reshape((-1,)) + _segmentation(prediction, values.shape[0], "labeled prediction") + if np.any(values > 5): + raise LidarGroundError("Ground annotation contains an unknown class") + reviewed = values != 0 + if not np.any(reviewed): + return { + "reviewed_points": 0, + "ground_iou": None, + "curb_recall": None, + "low_obstacle_recall": None, + "reflection_noise_rejection": None, + } + ground_truth = values == 1 + union = reviewed & (ground_truth | prediction.ground_mask) + intersection = ground_truth & prediction.ground_mask + return { + "reviewed_points": int(np.count_nonzero(reviewed)), + "ground_iou": _ratio( + int(np.count_nonzero(intersection)), + int(np.count_nonzero(union)), + ), + "curb_recall": _class_non_ground_recall(prediction, values, 2), + "low_obstacle_recall": _class_non_ground_recall(prediction, values, 3), + "reflection_noise_rejection": _class_unassigned_recall( + prediction, + values, + 4, + ), + } + + +def lidar_ground_benchmark_catalog_item( + benchmark: LidarGroundBenchmarkV1, +) -> dict[str, object]: + report = benchmark.report + return { + "benchmark_id": benchmark.benchmark_id, + "replay_pack_id": benchmark.identity["replay_pack_id"], + "session_id": benchmark.identity["session_id"], + "status": report["status"], + "frames": report["frames"], + "points": report["points"], + "input_domain": report["input_domain"], + "labels": report["labels"], + "current": report["current"], + "candidate": report["candidate"], + "comparison": report["comparison"], + "decision": report["decision"], + "created_at_utc": benchmark.manifest.get("created_at_utc"), + "authority": { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + + +def _validate_annotation_template(root: Path) -> None: + resolved = root.resolve(strict=True) + if ( + root.is_symlink() + or not resolved.is_dir() + or _ANNOTATION_TEMPLATE_ID.fullmatch(resolved.name) is None + ): + raise LidarGroundError("Ground annotation template id is invalid") + manifest = _read_json(resolved / LIDAR_GROUND_MANIFEST_NAME) + identity = _object(manifest.get("identity"), "annotation template identity") + identity_sha256 = manifest.get("identity_sha256") + if ( + manifest.get("schema_version") != LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA + or identity.get("schema_version") + != LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or resolved.name != f"ground-annotation-template-{identity_sha256}" + or manifest.get("ground_truth") is not False + or identity.get("review_status") != "unreviewed" + ): + raise LidarGroundError("Ground annotation template identity is invalid") + artifacts = _validate_artifacts(resolved, manifest.get("artifacts")) + arrays = np.load(artifacts["labels-template"], allow_pickle=False) + try: + required = { + "source_frame_index", + "point_capture_sequence", + "point_offsets", + "labels", + } + if ( + set(arrays.files) != required + or arrays["labels"].dtype != np.dtype("u1") + or np.any(arrays["labels"] != 0) + or _ground_logical_sha256(arrays) + != identity.get("logical_content_sha256") + ): + raise LidarGroundError("Ground annotation template content is invalid") + finally: + arrays.close() + + +def _validate_ground_arrays(arrays: Any, identity: Mapping[str, object]) -> None: + required = { + "point_offsets": np.dtype(" arrays["current_assigned"]) + or np.any(arrays["candidate_ground"] > arrays["candidate_assigned"]) + ): + raise LidarGroundError("Ground benchmark array shapes or values are invalid") + + +def _xyzi(value: npt.NDArray[np.float32]) -> npt.NDArray[np.float32]: + points = np.asarray(value, dtype=np.float32) + if points.ndim != 2 or points.shape[1] != 4 or not np.isfinite(points).all(): + raise LidarGroundError("Ground provider input must be finite XYZI") + return points + + +def _indices(value: npt.NDArray[np.int64], count: int, label: str) -> None: + if ( + value.size + and ( + np.any(value < 0) + or np.any(value >= count) + or np.unique(value).shape[0] != value.shape[0] + ) + ): + raise LidarGroundError(f"{label} indices are invalid") + + +def _segmentation(value: GroundSegmentation, count: int, label: str) -> None: + if ( + value.ground_mask.dtype != np.dtype(np.bool_) + or value.assigned_mask.dtype != np.dtype(np.bool_) + or value.ground_mask.shape != (count,) + or value.assigned_mask.shape != (count,) + or np.any(value.ground_mask & ~value.assigned_mask) + or not math.isfinite(value.latency_ms) + or value.latency_ms < 0 + ): + raise LidarGroundError(f"{label} segmentation is invalid") + + +def _class_non_ground_recall( + prediction: GroundSegmentation, + labels: npt.NDArray[np.uint8], + class_id: int, +) -> float | None: + class_mask = labels == class_id + return _ratio( + int(np.count_nonzero(class_mask & ~prediction.ground_mask)), + int(np.count_nonzero(class_mask)), + ) + + +def _class_unassigned_recall( + prediction: GroundSegmentation, + labels: npt.NDArray[np.uint8], + class_id: int, +) -> float | None: + class_mask = labels == class_id + return _ratio( + int(np.count_nonzero(class_mask & ~prediction.assigned_mask)), + int(np.count_nonzero(class_mask)), + ) + + +def _ratio(numerator: int, denominator: int) -> float | None: + return float(numerator / denominator) if denominator else None + + +def _nonnegative_int(value: object, label: str) -> int: + if ( + not isinstance(value, int) + or isinstance(value, bool) + or value < 0 + ): + raise LidarGroundError(f"{label} must be a non-negative integer") + return value + + +def _distribution(values: list[float]) -> dict[str, float | int | None]: + array = np.asarray(values, dtype=np.float64) + if not array.size: + return { + "sample_count": 0, + "minimum": None, + "mean": None, + "p50": None, + "p95": None, + "maximum": None, + } + if not np.isfinite(array).all(): + raise LidarGroundError("Ground benchmark distribution is non-finite") + return { + "sample_count": int(array.size), + "minimum": float(np.min(array)), + "mean": float(np.mean(array)), + "p50": float(np.quantile(array, 0.5)), + "p95": float(np.quantile(array, 0.95)), + "maximum": float(np.max(array)), + } + + +def _ground_logical_sha256(arrays: Mapping[str, Any]) -> str: + digest = hashlib.sha256() + for name in sorted(arrays): + value = np.asarray(arrays[name]) + digest.update(name.encode()) + digest.update(value.dtype.str.encode()) + digest.update(_canonical_json(list(value.shape))) + digest.update(np.ascontiguousarray(value).tobytes()) + return digest.hexdigest() + + +def _artifact_descriptor( + role: str, + path: Path, + media_type: str, +) -> dict[str, object]: + return { + "role": role, + "name": path.name, + "media_type": media_type, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _validate_artifacts( + root: Path, + value: object, +) -> dict[str, Path]: + if not isinstance(value, list) or not value: + raise LidarGroundError("Ground artifact list is invalid") + artifacts: dict[str, Path] = {} + for item in value: + descriptor = _object(item, "ground artifact") + role = descriptor.get("role") + name = descriptor.get("name") + if ( + not isinstance(role, str) + or not isinstance(name, str) + or Path(name).name != name + or role in artifacts + ): + raise LidarGroundError("Ground artifact descriptor is invalid") + path = root / name + try: + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise LidarGroundError("Ground artifact is unavailable") from exc + if ( + path.is_symlink() + or not path.is_file() + or resolved.parent != root + or descriptor.get("byte_length") != metadata.st_size + or descriptor.get("sha256") != _sha256(path) + ): + raise LidarGroundError("Ground artifact integrity check failed") + artifacts[role] = path + return artifacts + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or not all( + isinstance(key, str) for key in value + ): + raise LidarGroundError(f"{label} must be an object") + return value + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LidarGroundError("Ground JSON artifact is unavailable") from exc + return _object(value, "ground JSON") + + +def _write_json(path: Path, value: object) -> None: + path.write_bytes(_canonical_json(value) + b"\n") + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> str: + return ( + datetime.now(UTC) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 962c8cc..74e76a9 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -407,7 +407,14 @@ app.include_router( / "compute-experiments" / "lidar-replay-v2" / "packs" - ) + ), + ground_root_provider=lambda: ( + REPOSITORY_ROOT + / ".runtime" + / "compute-experiments" + / "lidar-ground-v1" + / "benchmarks" + ), ) ) diff --git a/src/k1link/web/lidar_api.py b/src/k1link/web/lidar_api.py index c63c5a1..453bfff 100644 --- a/src/k1link/web/lidar_api.py +++ b/src/k1link/web/lidar_api.py @@ -9,14 +9,21 @@ from typing import Any, Final from fastapi import APIRouter, HTTPException, Query from k1link.compute import ( + LidarGroundBenchmarkV1, + LidarGroundError, LidarReplayError, LidarReplayPackV2, + lidar_ground_benchmark_catalog_item, lidar_pack_catalog_item, lidar_pack_detail, ) LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1" +LIDAR_GROUND_CATALOG_SCHEMA: Final = ( + "missioncore.lidar-ground-benchmark-catalog/v1" +) _PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$") +_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$") RootProvider = Callable[[], Path | None] @@ -25,9 +32,15 @@ def configured_lidar_replay_root() -> Path | None: return Path(value).expanduser().absolute() if value else None +def configured_lidar_ground_root() -> Path | None: + value = os.environ.get("MISSIONCORE_LIDAR_GROUND_ROOT", "").strip() + return Path(value).expanduser().absolute() if value else None + + def build_lidar_router( *, root_provider: RootProvider = configured_lidar_replay_root, + ground_root_provider: RootProvider = configured_lidar_ground_root, ) -> APIRouter: router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"]) @@ -107,4 +120,95 @@ def build_lidar_router( detail="LiDAR replay pack не прошёл проверку целостности", ) from exc + @router.get("/ground-benchmarks") + def list_lidar_ground_benchmarks( + pack_id: str | None = Query(default=None), + limit: int = Query(default=20, ge=1, le=100), + ) -> dict[str, Any]: + if pack_id is not None and _PACK_ID.fullmatch(pack_id) is None: + raise HTTPException(status_code=404, detail="LiDAR replay pack не найден") + root = ground_root_provider() + if root is None or not root.is_dir(): + return { + "schema_version": LIDAR_GROUND_CATALOG_SCHEMA, + "configured": root is not None, + "items": [], + "valid_total": 0, + "invalid_total": 0, + "access": "read-only", + } + items: list[dict[str, object]] = [] + invalid_total = 0 + candidates = sorted( + ( + candidate + for candidate in root.iterdir() + if candidate.is_dir() + and _BENCHMARK_ID.fullmatch(candidate.name) is not None + ), + key=lambda candidate: candidate.stat().st_mtime_ns, + reverse=True, + ) + for candidate in candidates: + try: + benchmark = LidarGroundBenchmarkV1(candidate) + try: + if ( + pack_id is None + or benchmark.identity.get("replay_pack_id") == pack_id + ): + items.append( + lidar_ground_benchmark_catalog_item(benchmark) + ) + finally: + benchmark.close() + except (LidarGroundError, OSError): + invalid_total += 1 + return { + "schema_version": LIDAR_GROUND_CATALOG_SCHEMA, + "configured": True, + "items": items[:limit], + "valid_total": len(items), + "invalid_total": invalid_total, + "access": "read-only", + } + + @router.get("/ground-benchmarks/{benchmark_id}") + def get_lidar_ground_benchmark(benchmark_id: str) -> dict[str, object]: + if _BENCHMARK_ID.fullmatch(benchmark_id) is None: + raise HTTPException( + status_code=404, + detail="LiDAR ground benchmark не найден", + ) + root = ground_root_provider() + if root is None or not root.is_dir(): + raise HTTPException( + status_code=503, + detail="LiDAR ground storage не настроен", + ) + candidate = root / benchmark_id + if not candidate.is_dir(): + raise HTTPException( + status_code=404, + detail="LiDAR ground benchmark не найден", + ) + try: + benchmark = LidarGroundBenchmarkV1(candidate) + try: + return { + "schema_version": ( + "missioncore.lidar-ground-benchmark-detail/v1" + ), + "benchmark": lidar_ground_benchmark_catalog_item(benchmark), + "report": benchmark.report, + "access": "read-only", + } + finally: + benchmark.close() + except (LidarGroundError, OSError) as exc: + raise HTTPException( + status_code=409, + detail="LiDAR ground benchmark не прошёл проверку целостности", + ) from exc + return router diff --git a/tests/test_lidar_ground.py b/tests/test_lidar_ground.py new file mode 100644 index 0000000..f13e467 --- /dev/null +++ b/tests/test_lidar_ground.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from fastapi import APIRouter +from fastapi.routing import APIRoute + +from k1link.compute import ( + DEFAULT_GROUND_BENCHMARK_PROFILE, + K1_LIDAR_PACK_V2_PROFILE, + GroundSegmentation, + LidarGroundBenchmarkV1, + LidarGroundError, + LidarReplayPointFrame, + LidarReplayPoseFrame, + LocalPercentileGroundSegmenter, + PatchworkPPGroundSegmenter, + build_lidar_ground_annotation_template, + build_lidar_ground_benchmark, + score_ground_labels, +) +from k1link.web.lidar_api import build_lidar_router + + +class _Replay: + def __init__(self) -> None: + self.pack_id = f"lidar-replay-pack-{'a' * 64}" + self.identity = { + "logical_content_sha256": "b" * 64, + "session_id": "synthetic-ground-session", + } + self.profile = K1_LIDAR_PACK_V2_PROFILE + self._points = ( + np.asarray( + [ + [-1.0, -1.0, 0.00], + [-0.5, -1.0, 0.02], + [0.0, -1.0, 0.01], + [0.5, -1.0, 0.03], + [1.0, -1.0, 0.00], + [-1.0, 0.0, 0.01], + [-0.5, 0.0, 0.02], + [0.0, 0.0, 0.04], + [0.5, 0.0, 0.35], + [1.0, 0.0, 0.70], + ], + dtype=np.float64, + ), + np.asarray( + [ + [-1.0, -1.0, 0.01], + [-0.5, -1.0, 0.01], + [0.0, -1.0, 0.02], + [0.5, -1.0, 0.02], + [1.0, -1.0, 0.01], + [-1.0, 0.0, 0.02], + [-0.5, 0.0, 0.01], + [0.0, 0.0, 0.03], + [0.5, 0.0, 0.40], + [1.0, 0.0, 0.80], + ], + dtype=np.float64, + ), + ) + self.arrays = { + "point_offsets": np.asarray([0, 10, 20], dtype=" int: + return 2 + + @property + def pose_frame_count(self) -> int: + return 2 + + @property + def point_count(self) -> int: + return 20 + + def point_frame(self, index: int) -> LidarReplayPointFrame: + xyz = self._points[index] + return LidarReplayPointFrame( + capture_sequence=1 + index * 2, + payload_bytes=100, + received_at_epoch_ns=2_000_000_000 + index * 100_000_000, + received_monotonic_ns=1_000_000_000 + index * 100_000_000, + header_seq=10 + index, + header_stamp=100 + index, + scaler=1000, + raw_xyz=(xyz * 1000).astype(np.int64), + xyz_map=xyz, + rgbi=np.full(10, 0xFFFFFF80, dtype=np.uint32), + intensity=np.full(10, 128, dtype=np.uint8), + ) + + def pose_frame(self, index: int) -> LidarReplayPoseFrame: + return LidarReplayPoseFrame( + capture_sequence=2 + index * 2, + payload_bytes=80, + received_at_epoch_ns=2_001_000_000 + index * 100_000_000, + received_monotonic_ns=1_001_000_000 + index * 100_000_000, + header_seq=20 + index, + header_stamp=200 + index, + header_scaler=1000, + pose_stamp=201 + index, + position_map=(0.0, 0.0, 0.0), + orientation_map_from_lidar=(0.0, 0.0, 0.0, 1.0), + distance=0.0, + pose_accuracy=0.001, + ) + + +class _Candidate: + @property + def identity(self) -> dict[str, object]: + return { + "provider_id": "test-patchwork/v1", + "binary_sha256": "c" * 64, + } + + def segment(self, xyzi: np.ndarray) -> GroundSegmentation: + ground = xyzi[:, 2] <= 0.025 + assigned = np.ones(xyzi.shape[0], dtype=np.bool_) + return GroundSegmentation(ground, assigned, 0.5) + + +def _endpoint(router: APIRouter, path: str) -> object: + for route in router.routes: + if isinstance(route, APIRoute) and route.path == path and "GET" in route.methods: + return route.endpoint + raise AssertionError(f"GET {path} route is missing") + + +def test_local_percentile_ground_is_point_aligned_and_non_mutating() -> None: + replay = _Replay() + xyz = replay._points[0] + xyzi = np.column_stack((xyz, np.ones(xyz.shape[0]))).astype(np.float32) + unchanged = xyzi.copy() + + result = LocalPercentileGroundSegmenter( + profile=DEFAULT_GROUND_BENCHMARK_PROFILE + ).segment(xyzi) + + assert result.ground_mask.shape == (10,) + assert result.assigned_mask.all() + assert 0 < np.count_nonzero(result.ground_mask) < 10 + np.testing.assert_array_equal(xyzi, unchanged) + + +def test_ground_benchmark_is_immutable_diagnostic_evidence(tmp_path: Path) -> None: + replay = _Replay() + output = build_lidar_ground_benchmark( + replay, # type: ignore[arg-type] + tmp_path / "benchmarks", + patchwork=_Candidate(), + ) + result = LidarGroundBenchmarkV1(output) + try: + assert result.report["status"] == "diagnostic-only" + assert result.report["input_domain"]["accepted"] is False + assert result.report["labels"]["metrics_available"] is False + assert ( + result.report["decision"]["status"] + == "do-not-promote-on-current-vendor-map" + ) + assert result.arrays["current_ground"].shape == (20,) + assert result.arrays["candidate_ground"].shape == (20,) + finally: + result.close() + + assert ( + build_lidar_ground_benchmark( + replay, # type: ignore[arg-type] + tmp_path / "benchmarks", + patchwork=_Candidate(), + ) + == output + ) + + manifest_path = output / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["identity"]["points"] = 21 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(LidarGroundError, match="identity"): + LidarGroundBenchmarkV1(output) + + +def test_annotation_template_starts_all_ignore_and_never_ground_truth( + tmp_path: Path, +) -> None: + replay = _Replay() + output = build_lidar_ground_annotation_template( + replay, # type: ignore[arg-type] + tmp_path / "annotations", + requested_frames=2, + ) + manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) + arrays = np.load(output / "labels-template.npz", allow_pickle=False) + try: + assert manifest["ground_truth"] is False + assert manifest["identity"]["review_status"] == "unreviewed" + assert arrays["labels"].shape == (20,) + assert not np.any(arrays["labels"]) + finally: + arrays.close() + + +def test_ground_api_is_read_only_path_free_and_pack_filtered( + tmp_path: Path, +) -> None: + replay = _Replay() + root = tmp_path / "benchmarks" + output = build_lidar_ground_benchmark( + replay, # type: ignore[arg-type] + root, + patchwork=_Candidate(), + ) + router = build_lidar_router( + root_provider=lambda: None, + ground_root_provider=lambda: root, + ) + catalog_route = _endpoint(router, "/api/v1/lidar/ground-benchmarks") + detail_route = _endpoint( + router, + "/api/v1/lidar/ground-benchmarks/{benchmark_id}", + ) + + catalog = catalog_route( # type: ignore[operator] + pack_id=replay.pack_id, + limit=20, + ) + detail = detail_route(benchmark_id=output.name) # type: ignore[operator] + + assert ( + catalog["schema_version"] + == "missioncore.lidar-ground-benchmark-catalog/v1" + ) + assert catalog["valid_total"] == 1 + assert catalog["items"][0]["decision"]["production_promotion"] is False + assert detail["benchmark"]["benchmark_id"] == output.name + assert detail["access"] == "read-only" + assert str(tmp_path) not in repr({"catalog": catalog, "detail": detail}) + + +def test_reviewed_ground_metrics_keep_ignore_out_of_denominators() -> None: + prediction = GroundSegmentation( + ground_mask=np.asarray([True, True, False, False, False, False]), + assigned_mask=np.asarray([True, True, True, True, False, True]), + latency_ms=1.0, + ) + labels = np.asarray([1, 2, 2, 3, 4, 0], dtype=np.uint8) + + metrics = score_ground_labels(prediction, labels) + + assert metrics["reviewed_points"] == 5 + assert metrics["ground_iou"] == pytest.approx(0.5) + assert metrics["curb_recall"] == pytest.approx(0.5) + assert metrics["low_obstacle_recall"] == pytest.approx(1.0) + assert metrics["reflection_noise_rejection"] == pytest.approx(1.0) + + +def test_patchwork_adapter_records_binary_and_rejects_overlapping_indices( + tmp_path: Path, +) -> None: + binary = tmp_path / "pypatchworkpp.so" + binary.write_bytes(b"synthetic-binding") + + class Parameters: + pass + + class Estimator: + def __init__(self, _params: object) -> None: + pass + + def estimateGround(self, _points: np.ndarray) -> None: + pass + + def getGroundIndices(self) -> list[int]: + return [0, 1] + + def getNongroundIndices(self) -> list[int]: + return [1, 2] + + module = SimpleNamespace( + __file__=str(binary), + __version__="test", + Parameters=Parameters, + patchworkpp=Estimator, + ) + adapter = PatchworkPPGroundSegmenter( # type: ignore[arg-type] + module, + DEFAULT_GROUND_BENCHMARK_PROFILE, + ) + assert adapter.identity["binary_sha256"] + with pytest.raises(LidarGroundError, match="assigned one point twice"): + adapter.segment(np.zeros((3, 4), dtype=np.float32))