feat(lidar): add ground segmentation diagnostic benchmark
This commit is contained in:
DCCONSTRUCTIONS
parent
e4fdd9fa20
commit
75a3e669d9
|
|
@ -59,6 +59,62 @@ export interface LidarReplayDetail {
|
|||
stages: LidarStageReadiness[];
|
||||
}
|
||||
|
||||
export interface LidarGroundProviderSummary {
|
||||
providerId: string;
|
||||
sourceCommit: string | null;
|
||||
}
|
||||
|
||||
export interface LidarGroundBenchmark {
|
||||
benchmarkId: string;
|
||||
replayPackId: string;
|
||||
sessionId: string;
|
||||
status: "diagnostic-only";
|
||||
frames: number;
|
||||
points: number;
|
||||
inputDomain: {
|
||||
accepted: false;
|
||||
representation: string;
|
||||
physicalSensorHeightKnown: boolean;
|
||||
sensorScanGeometryKnown: boolean;
|
||||
reason: string;
|
||||
};
|
||||
labels: {
|
||||
status: "missing-independent-review";
|
||||
metricsAvailable: false;
|
||||
};
|
||||
current: {
|
||||
provider: LidarGroundProviderSummary;
|
||||
groundFraction: LidarDistribution;
|
||||
assignedFraction: LidarDistribution;
|
||||
latencyMs: LidarDistribution;
|
||||
};
|
||||
candidate: {
|
||||
provider: LidarGroundProviderSummary;
|
||||
groundFraction: LidarDistribution;
|
||||
assignedFraction: LidarDistribution;
|
||||
latencyMs: LidarDistribution;
|
||||
};
|
||||
comparison: {
|
||||
algorithmGroundIou: LidarDistribution;
|
||||
groundDisagreementFraction: LidarDistribution;
|
||||
isAccuracyMetric: false;
|
||||
};
|
||||
decision: {
|
||||
status: "do-not-promote-on-current-vendor-map";
|
||||
productionPromotion: false;
|
||||
reasons: string[];
|
||||
nextGate: string;
|
||||
};
|
||||
createdAtUtc: string | null;
|
||||
}
|
||||
|
||||
export interface LidarGroundBenchmarkCatalog {
|
||||
configured: boolean;
|
||||
validTotal: number;
|
||||
invalidTotal: number;
|
||||
items: LidarGroundBenchmark[];
|
||||
}
|
||||
|
||||
export class LidarReplayContractError extends Error {}
|
||||
|
||||
export class LidarReplayApiError extends Error {
|
||||
|
|
@ -73,8 +129,10 @@ type LidarFetch = (
|
|||
) => Promise<Response>;
|
||||
|
||||
const SAFE_PACK_ID = /^lidar-replay-pack-[a-f0-9]{64}$/;
|
||||
const SAFE_GROUND_BENCHMARK_ID = /^ground-benchmark-[a-f0-9]{64}$/;
|
||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$/;
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const GIT_SHA1 = /^[a-f0-9]{40}$/;
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
|
|
@ -258,6 +316,139 @@ export function parseLidarReplayDetail(value: unknown): LidarReplayDetail {
|
|||
};
|
||||
}
|
||||
|
||||
function groundProvider(
|
||||
value: unknown,
|
||||
label: string,
|
||||
): LidarGroundProviderSummary {
|
||||
const source = record(value, label);
|
||||
return {
|
||||
providerId: string(source.provider_id, `${label}.provider_id`, SAFE_ID),
|
||||
sourceCommit:
|
||||
source.source_commit === undefined || source.source_commit === null
|
||||
? null
|
||||
: string(source.source_commit, `${label}.source_commit`, GIT_SHA1),
|
||||
};
|
||||
}
|
||||
|
||||
function groundBranch(
|
||||
value: unknown,
|
||||
label: string,
|
||||
): LidarGroundBenchmark["current"] {
|
||||
const source = record(value, label);
|
||||
return {
|
||||
provider: groundProvider(source.provider, `${label}.provider`),
|
||||
groundFraction: distribution(
|
||||
source.ground_fraction,
|
||||
`${label}.ground_fraction`,
|
||||
),
|
||||
assignedFraction: distribution(
|
||||
source.assigned_fraction,
|
||||
`${label}.assigned_fraction`,
|
||||
),
|
||||
latencyMs: distribution(source.latency_ms, `${label}.latency_ms`),
|
||||
};
|
||||
}
|
||||
|
||||
function groundBenchmark(value: unknown): LidarGroundBenchmark {
|
||||
const source = record(value, "LiDAR ground benchmark");
|
||||
if (source.status !== "diagnostic-only") {
|
||||
throw new LidarReplayContractError("Ground benchmark status несовместим");
|
||||
}
|
||||
const inputDomain = record(source.input_domain, "input_domain");
|
||||
const labels = record(source.labels, "labels");
|
||||
const comparison = record(source.comparison, "comparison");
|
||||
const decision = record(source.decision, "decision");
|
||||
if (
|
||||
inputDomain.accepted !== false
|
||||
|| labels.status !== "missing-independent-review"
|
||||
|| labels.metrics_available !== false
|
||||
|| comparison.is_accuracy_metric !== false
|
||||
|| decision.status !== "do-not-promote-on-current-vendor-map"
|
||||
|| decision.production_promotion !== false
|
||||
) {
|
||||
throw new LidarReplayContractError(
|
||||
"Ground benchmark завышает readiness или accuracy",
|
||||
);
|
||||
}
|
||||
return {
|
||||
benchmarkId: string(
|
||||
source.benchmark_id,
|
||||
"benchmark_id",
|
||||
SAFE_GROUND_BENCHMARK_ID,
|
||||
),
|
||||
replayPackId: string(source.replay_pack_id, "replay_pack_id", SAFE_PACK_ID),
|
||||
sessionId: string(source.session_id, "session_id", SAFE_ID),
|
||||
status: "diagnostic-only",
|
||||
frames: integer(source.frames, "frames"),
|
||||
points: integer(source.points, "points"),
|
||||
inputDomain: {
|
||||
accepted: false,
|
||||
representation: string(
|
||||
inputDomain.representation,
|
||||
"input_domain.representation",
|
||||
SAFE_ID,
|
||||
),
|
||||
physicalSensorHeightKnown: boolean(
|
||||
inputDomain.physical_sensor_height_known,
|
||||
"input_domain.physical_sensor_height_known",
|
||||
),
|
||||
sensorScanGeometryKnown: boolean(
|
||||
inputDomain.sensor_scan_geometry_known,
|
||||
"input_domain.sensor_scan_geometry_known",
|
||||
),
|
||||
reason: string(inputDomain.reason, "input_domain.reason"),
|
||||
},
|
||||
labels: {
|
||||
status: "missing-independent-review",
|
||||
metricsAvailable: false,
|
||||
},
|
||||
current: groundBranch(source.current, "current"),
|
||||
candidate: groundBranch(source.candidate, "candidate"),
|
||||
comparison: {
|
||||
algorithmGroundIou: distribution(
|
||||
comparison.algorithm_to_algorithm_ground_iou,
|
||||
"comparison.algorithm_ground_iou",
|
||||
),
|
||||
groundDisagreementFraction: distribution(
|
||||
comparison.ground_disagreement_fraction,
|
||||
"comparison.ground_disagreement_fraction",
|
||||
),
|
||||
isAccuracyMetric: false,
|
||||
},
|
||||
decision: {
|
||||
status: "do-not-promote-on-current-vendor-map",
|
||||
productionPromotion: false,
|
||||
reasons: array(decision.reasons, "decision.reasons").map((reason) =>
|
||||
string(reason, "decision.reason")
|
||||
),
|
||||
nextGate: string(decision.next_gate, "decision.next_gate"),
|
||||
},
|
||||
createdAtUtc:
|
||||
source.created_at_utc === null || source.created_at_utc === undefined
|
||||
? null
|
||||
: string(source.created_at_utc, "created_at_utc"),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseLidarGroundBenchmarkCatalog(
|
||||
value: unknown,
|
||||
): LidarGroundBenchmarkCatalog {
|
||||
const source = record(value, "LiDAR ground catalog");
|
||||
if (
|
||||
source.schema_version
|
||||
!== "missioncore.lidar-ground-benchmark-catalog/v1"
|
||||
|| source.access !== "read-only"
|
||||
) {
|
||||
throw new LidarReplayContractError("LiDAR ground catalog несовместим");
|
||||
}
|
||||
return {
|
||||
configured: boolean(source.configured, "configured"),
|
||||
validTotal: integer(source.valid_total, "valid_total"),
|
||||
invalidTotal: integer(source.invalid_total, "invalid_total"),
|
||||
items: array(source.items, "items").map(groundBenchmark),
|
||||
};
|
||||
}
|
||||
|
||||
async function responseJson(
|
||||
response: Response,
|
||||
fallback: string,
|
||||
|
|
@ -309,3 +500,24 @@ export async function fetchLidarReplayDetail(
|
|||
await responseJson(response, "Не удалось получить LiDAR quality report."),
|
||||
);
|
||||
}
|
||||
|
||||
export async function fetchLidarGroundBenchmarks(
|
||||
packId: string,
|
||||
options: { signal?: AbortSignal; fetcher?: LidarFetch } = {},
|
||||
): Promise<LidarGroundBenchmarkCatalog> {
|
||||
if (!SAFE_PACK_ID.test(packId)) {
|
||||
throw new LidarReplayContractError("Некорректный LiDAR pack id");
|
||||
}
|
||||
const fetcher = options.fetcher ?? fetch;
|
||||
const response = await fetcher(
|
||||
`/api/v1/lidar/ground-benchmarks?pack_id=${encodeURIComponent(packId)}&limit=20`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json" },
|
||||
signal: options.signal,
|
||||
},
|
||||
);
|
||||
return parseLidarGroundBenchmarkCatalog(
|
||||
await responseJson(response, "Не удалось получить LiDAR ground benchmark."),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,10 @@
|
|||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.lidar-ground-metrics {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.pipeline-strip {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
|
|
@ -162,6 +166,11 @@
|
|||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.lidar-ground-metrics,
|
||||
.lidar-ground-gates {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.polygon-run-identity dl {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -954,6 +954,76 @@
|
|||
margin-top: 0.8rem;
|
||||
}
|
||||
|
||||
.lidar-ground-benchmark {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.lidar-ground-metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.lidar-ground-metrics > div {
|
||||
display: grid;
|
||||
gap: 0.3rem;
|
||||
border: 1px solid var(--station-hairline);
|
||||
border-radius: 0.8rem;
|
||||
background: rgb(255 255 255 / 0.025);
|
||||
padding: 0.75rem;
|
||||
}
|
||||
|
||||
.lidar-ground-metrics span,
|
||||
.lidar-ground-metrics small,
|
||||
.lidar-ground-gates p,
|
||||
.lidar-ground-footer,
|
||||
.lidar-ground-empty {
|
||||
color: var(--nodedc-text-muted);
|
||||
font-size: 0.62rem;
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
.lidar-ground-metrics strong {
|
||||
color: var(--nodedc-text-primary);
|
||||
font-size: 1.15rem;
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.lidar-ground-gates {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
margin-top: 0.75rem;
|
||||
}
|
||||
|
||||
.lidar-ground-gates > div {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 0.55rem;
|
||||
border-left: 1px solid var(--station-hairline);
|
||||
padding-left: 0.7rem;
|
||||
}
|
||||
|
||||
.lidar-ground-gates p,
|
||||
.lidar-ground-empty {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.lidar-ground-footer {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: space-between;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.85rem;
|
||||
border-top: 1px solid var(--station-hairline);
|
||||
padding-top: 0.7rem;
|
||||
}
|
||||
|
||||
.lidar-ground-empty {
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.overview-grid {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1.55fr) minmax(19rem, 0.75fr);
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ import {
|
|||
} from "@nodedc/ui-react";
|
||||
|
||||
import {
|
||||
fetchLidarGroundBenchmarks,
|
||||
fetchLidarReplayCatalog,
|
||||
fetchLidarReplayDetail,
|
||||
type LidarGroundBenchmark,
|
||||
type LidarReplayCatalog,
|
||||
type LidarReplayDetail,
|
||||
type LidarStageReadiness,
|
||||
|
|
@ -53,6 +55,8 @@ export function LidarQualityWorkspace({
|
|||
}) {
|
||||
const [catalog, setCatalog] = useState<LidarReplayCatalog | null>(null);
|
||||
const [detail, setDetail] = useState<LidarReplayDetail | null>(null);
|
||||
const [groundBenchmark, setGroundBenchmark] =
|
||||
useState<LidarGroundBenchmark | null>(null);
|
||||
const [selectedPackId, setSelectedPackId] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
|
@ -72,17 +76,25 @@ export function LidarQualityWorkspace({
|
|||
const target = selectedPackId ?? nextCatalog.items[0]?.packId ?? null;
|
||||
if (!target) {
|
||||
setDetail(null);
|
||||
setGroundBenchmark(null);
|
||||
return;
|
||||
}
|
||||
const nextDetail = await fetchLidarReplayDetail(target, {
|
||||
signal: controller.signal,
|
||||
});
|
||||
const [nextDetail, groundCatalog] = await Promise.all([
|
||||
fetchLidarReplayDetail(target, {
|
||||
signal: controller.signal,
|
||||
}),
|
||||
fetchLidarGroundBenchmarks(target, {
|
||||
signal: controller.signal,
|
||||
}),
|
||||
]);
|
||||
if (controller.signal.aborted) return;
|
||||
setSelectedPackId(target);
|
||||
setDetail(nextDetail);
|
||||
setGroundBenchmark(groundCatalog.items[0] ?? null);
|
||||
} catch (loadError) {
|
||||
if (controller.signal.aborted) return;
|
||||
setDetail(null);
|
||||
setGroundBenchmark(null);
|
||||
setError(errorMessage(loadError));
|
||||
} finally {
|
||||
if (!controller.signal.aborted) setLoading(false);
|
||||
|
|
@ -226,6 +238,118 @@ export function LidarQualityWorkspace({
|
|||
</GlassSurface>
|
||||
</div>
|
||||
|
||||
<GlassSurface className="lidar-ground-benchmark" padding="lg">
|
||||
<header className="panel-heading">
|
||||
<div>
|
||||
<span className="section-eyebrow">GROUND A/B · L2</span>
|
||||
<h2>Текущий heuristic против Patchwork++</h2>
|
||||
</div>
|
||||
<StatusBadge tone={groundBenchmark ? "warning" : "neutral"}>
|
||||
{groundBenchmark ? "Diagnostic only" : "Нет результата"}
|
||||
</StatusBadge>
|
||||
</header>
|
||||
{groundBenchmark ? (
|
||||
<>
|
||||
<section
|
||||
className="lidar-ground-metrics"
|
||||
aria-label="Ground segmentation A/B"
|
||||
>
|
||||
<div>
|
||||
<span>Текущий ground p50</span>
|
||||
<strong>
|
||||
{formatFraction(
|
||||
groundBenchmark.current.groundFraction.p50,
|
||||
)}
|
||||
</strong>
|
||||
<small>
|
||||
{formatNumber(
|
||||
groundBenchmark.current.latencyMs.p95,
|
||||
2,
|
||||
)}{" "}
|
||||
мс p95
|
||||
</small>
|
||||
</div>
|
||||
<div>
|
||||
<span>Patchwork++ ground p50</span>
|
||||
<strong>
|
||||
{formatFraction(
|
||||
groundBenchmark.candidate.groundFraction.p50,
|
||||
)}
|
||||
</strong>
|
||||
<small>
|
||||
{formatNumber(
|
||||
groundBenchmark.candidate.latencyMs.p95,
|
||||
2,
|
||||
)}{" "}
|
||||
мс p95
|
||||
</small>
|
||||
</div>
|
||||
<div>
|
||||
<span>Algorithm IoU p50</span>
|
||||
<strong>
|
||||
{formatFraction(
|
||||
groundBenchmark.comparison.algorithmGroundIou.p50,
|
||||
)}
|
||||
</strong>
|
||||
<small>Не является accuracy</small>
|
||||
</div>
|
||||
<div>
|
||||
<span>Disagreement p50</span>
|
||||
<strong>
|
||||
{formatFraction(
|
||||
groundBenchmark.comparison
|
||||
.groundDisagreementFraction.p50,
|
||||
)}
|
||||
</strong>
|
||||
<small>
|
||||
{groundBenchmark.frames.toLocaleString("ru-RU")} кадров
|
||||
</small>
|
||||
</div>
|
||||
</section>
|
||||
<div className="lidar-ground-gates">
|
||||
<div>
|
||||
<StatusBadge tone="danger">
|
||||
Входной контракт не принят
|
||||
</StatusBadge>
|
||||
<p>
|
||||
Patchwork++ ожидает sensor-centric scan и физическую высоту
|
||||
сенсора; текущий point feed является vendor-mapped increment.
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<StatusBadge tone="warning">Разметка не принята</StatusBadge>
|
||||
<p>
|
||||
IoU, curb recall, low-obstacle recall и reflection-noise
|
||||
rejection появятся только после независимого human review.
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<StatusBadge tone="danger">Не продвигать</StatusBadge>
|
||||
<p>
|
||||
Следующий gate: human-reviewed annotation subset или
|
||||
принятый raw sensor scan с физической высотой сенсора.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<footer className="lidar-ground-footer">
|
||||
<span>
|
||||
{groundBenchmark.current.provider.providerId}
|
||||
</span>
|
||||
<span>
|
||||
{groundBenchmark.candidate.provider.providerId}
|
||||
{groundBenchmark.candidate.provider.sourceCommit
|
||||
? ` · ${groundBenchmark.candidate.provider.sourceCommit.slice(0, 12)}`
|
||||
: ""}
|
||||
</span>
|
||||
</footer>
|
||||
</>
|
||||
) : (
|
||||
<p className="lidar-ground-empty">
|
||||
Для выбранного replay pack ещё нет проверенного ground benchmark.
|
||||
</p>
|
||||
)}
|
||||
</GlassSurface>
|
||||
|
||||
<GlassSurface className="lidar-pack-catalog" padding="lg">
|
||||
<header className="panel-heading">
|
||||
<div>
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ import { createServer } from "vite";
|
|||
let server;
|
||||
let parseLidarReplayCatalog;
|
||||
let parseLidarReplayDetail;
|
||||
let parseLidarGroundBenchmarkCatalog;
|
||||
let fetchLidarReplayCatalog;
|
||||
let fetchLidarReplayDetail;
|
||||
let fetchLidarGroundBenchmarks;
|
||||
let LidarReplayContractError;
|
||||
let workspaceById;
|
||||
|
||||
|
|
@ -20,8 +22,10 @@ before(async () => {
|
|||
({
|
||||
parseLidarReplayCatalog,
|
||||
parseLidarReplayDetail,
|
||||
parseLidarGroundBenchmarkCatalog,
|
||||
fetchLidarReplayCatalog,
|
||||
fetchLidarReplayDetail,
|
||||
fetchLidarGroundBenchmarks,
|
||||
LidarReplayContractError,
|
||||
} = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts"));
|
||||
({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts"));
|
||||
|
|
@ -130,6 +134,90 @@ function detail() {
|
|||
};
|
||||
}
|
||||
|
||||
function groundCatalog(overrides = {}) {
|
||||
const provider = (providerId, sourceCommit = undefined) => ({
|
||||
provider_id: providerId,
|
||||
source_commit: sourceCommit,
|
||||
});
|
||||
const branch = (providerValue, groundP50, latencyP95) => ({
|
||||
provider: providerValue,
|
||||
ground_fraction: {
|
||||
...distribution(),
|
||||
p50: groundP50,
|
||||
},
|
||||
assigned_fraction: {
|
||||
...distribution(),
|
||||
minimum: 1,
|
||||
mean: 1,
|
||||
p50: 1,
|
||||
p95: 1,
|
||||
maximum: 1,
|
||||
},
|
||||
latency_ms: {
|
||||
...distribution(),
|
||||
p95: latencyP95,
|
||||
},
|
||||
});
|
||||
return {
|
||||
schema_version: "missioncore.lidar-ground-benchmark-catalog/v1",
|
||||
configured: true,
|
||||
valid_total: 1,
|
||||
invalid_total: 0,
|
||||
access: "read-only",
|
||||
items: [{
|
||||
benchmark_id: `ground-benchmark-${"c".repeat(64)}`,
|
||||
replay_pack_id: packId,
|
||||
session_id: "20260719T220917Z_viewer_live",
|
||||
status: "diagnostic-only",
|
||||
frames: 66,
|
||||
points: 226963,
|
||||
input_domain: {
|
||||
accepted: false,
|
||||
representation: "vendor-map-increment",
|
||||
physical_sensor_height_known: false,
|
||||
sensor_scan_geometry_known: false,
|
||||
reason: "Patchwork++ expects sensor-centric scans.",
|
||||
},
|
||||
labels: {
|
||||
status: "missing-independent-review",
|
||||
metrics_available: false,
|
||||
},
|
||||
current: branch(
|
||||
provider("missioncore-local-percentile-ground/v1"),
|
||||
0.183,
|
||||
10.7,
|
||||
),
|
||||
candidate: branch(
|
||||
provider(
|
||||
"patchworkpp/v1.4.1",
|
||||
"3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||
),
|
||||
0.005,
|
||||
0.29,
|
||||
),
|
||||
comparison: {
|
||||
algorithm_to_algorithm_ground_iou: {
|
||||
...distribution(),
|
||||
p50: 0.029,
|
||||
},
|
||||
ground_disagreement_fraction: {
|
||||
...distribution(),
|
||||
p50: 0.179,
|
||||
},
|
||||
is_accuracy_metric: false,
|
||||
},
|
||||
decision: {
|
||||
status: "do-not-promote-on-current-vendor-map",
|
||||
production_promotion: false,
|
||||
reasons: ["candidate input domain is not accepted"],
|
||||
next_gate: "human-reviewed annotation subset",
|
||||
},
|
||||
created_at_utc: "2026-07-25T01:00:00Z",
|
||||
}],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function jsonResponse(payload, status = 200) {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
|
|
@ -157,22 +245,48 @@ test("LiDAR contract refuses replay that did not pass equivalence", () => {
|
|||
);
|
||||
});
|
||||
|
||||
test("ground benchmark stays diagnostic until input and labels are accepted", () => {
|
||||
const parsed = parseLidarGroundBenchmarkCatalog(groundCatalog());
|
||||
|
||||
assert.equal(parsed.items[0].status, "diagnostic-only");
|
||||
assert.equal(parsed.items[0].inputDomain.accepted, false);
|
||||
assert.equal(parsed.items[0].labels.metricsAvailable, false);
|
||||
assert.equal(parsed.items[0].candidate.groundFraction.p50, 0.005);
|
||||
assert.equal(parsed.items[0].decision.productionPromotion, false);
|
||||
|
||||
const promoted = groundCatalog();
|
||||
promoted.items[0].decision.production_promotion = true;
|
||||
assert.throws(
|
||||
() => parseLidarGroundBenchmarkCatalog(promoted),
|
||||
LidarReplayContractError,
|
||||
);
|
||||
});
|
||||
|
||||
test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => {
|
||||
const calls = [];
|
||||
const fetcher = async (input, init) => {
|
||||
calls.push({ input: String(input), method: init?.method });
|
||||
if (String(input).includes("ground-benchmarks")) {
|
||||
return jsonResponse(groundCatalog());
|
||||
}
|
||||
return String(input).includes(packId)
|
||||
? jsonResponse(detail())
|
||||
: jsonResponse(catalog());
|
||||
};
|
||||
const parsedCatalog = await fetchLidarReplayCatalog({ fetcher });
|
||||
const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher });
|
||||
const ground = await fetchLidarGroundBenchmarks(packId, { fetcher });
|
||||
|
||||
assert.equal(parsedCatalog.validTotal, 1);
|
||||
assert.equal(parsedDetail.pack.packId, packId);
|
||||
assert.equal(ground.validTotal, 1);
|
||||
assert.deepEqual(calls, [
|
||||
{ input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" },
|
||||
{ input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" },
|
||||
{
|
||||
input: `/api/v1/lidar/ground-benchmarks?pack_id=${packId}&limit=20`,
|
||||
method: "GET",
|
||||
},
|
||||
]);
|
||||
assert.equal(workspaceById("lidar-quality").root, "data");
|
||||
assert.equal(workspaceById("lidar-quality").kind, "lidar-quality");
|
||||
|
|
|
|||
|
|
@ -14,6 +14,16 @@ equivalence report. The React surface reads those reports through the read-only
|
|||
`/api/v1/lidar/replay-packs` boundary; CUDA/TensorRT and ROS do not move into the
|
||||
browser.
|
||||
|
||||
`missioncore.lidar-ground-benchmark/v1` extends the same boundary for L2. It
|
||||
keeps point-aligned ground/assigned masks for the current local-percentile
|
||||
proposal and a candidate provider, exact provider/source/binary identities,
|
||||
host latency and algorithm disagreement. The first official Patchwork++ v1.4.1
|
||||
run is diagnostic-only: current K1 evidence is a vendor-map increment, not the
|
||||
sensor-centric scan and physical-height contract Patchwork++ expects. The
|
||||
read-only `/api/v1/lidar/ground-benchmarks` surface therefore publishes
|
||||
`production_promotion=false` until an independent annotation generation or an
|
||||
admitted raw scan closes the input gate.
|
||||
|
||||
## Boundary
|
||||
|
||||
```text
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
# LiDAR worker: product value, evidence boundary and implementation roadmap
|
||||
|
||||
Date: 2026-07-25
|
||||
Status: accepted architecture plan; L0 and L1 implemented
|
||||
Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete,
|
||||
independent labels pending
|
||||
Scope: real scanner records, replay and future live shadow processing
|
||||
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
||||
|
||||
|
|
@ -111,7 +112,7 @@ and training-domain fit.
|
|||
|
||||
| Component | Correct use | Decision |
|
||||
| --- | --- | --- |
|
||||
| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | First non-neural geometry baseline |
|
||||
| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | v1.4.1 benchmarked; do not promote on the current vendor-map feed |
|
||||
| [Autoware CenterPoint](https://github.com/autowarefoundation/autoware_universe/tree/main/perception/autoware_lidar_centerpoint) | Mature ROS 2/TensorRT 3D detection and multi-frame reference | Second detector baseline after PointPillars |
|
||||
| [MMDetection3D](https://github.com/open-mmlab/mmdetection3d) | Training/evaluation harness and dataset adapters | Laboratory only |
|
||||
| [OpenPCDet](https://github.com/open-mmlab/OpenPCDet) | Alternative LiDAR detector benchmark/model zoo | Laboratory only; not the production runtime |
|
||||
|
|
@ -192,17 +193,43 @@ Accepted real slice:
|
|||
The long interval tail is evidence to investigate in the scanner/transport
|
||||
quality line. It is not repaired or hidden by replay.
|
||||
|
||||
### L2 — geometric baseline
|
||||
### L2 — geometric baseline — diagnostic A/B complete, accuracy gate open
|
||||
|
||||
- [ ] Run Patchwork++ over the frozen XYZI slice.
|
||||
- [ ] Retain ground and non-ground outputs separately; never delete raw points.
|
||||
- [ ] Label a small independent ground/obstacle evaluation set in CVAT or an
|
||||
equivalent accepted annotation workspace.
|
||||
- [ ] Measure ground IoU, curb/low-obstacle recall, reflection-noise rejection
|
||||
and CPU/GPU latency.
|
||||
- [ ] Compare against the current heuristic ground proposal branch.
|
||||
- [x] Pin and run official Patchwork++ v1.4.1 at source commit
|
||||
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`.
|
||||
- [x] Compare it against a full-frame, point-aligned extension of the current
|
||||
E19 local-percentile ground proposal.
|
||||
- [x] Retain separate current/candidate ground and assigned masks for every
|
||||
source point; raw replay remains unchanged.
|
||||
- [x] Seal latency, ground-fraction, algorithm-IoU and disagreement
|
||||
distributions in `missioncore.lidar-ground-benchmark/v1`.
|
||||
- [x] Create an immutable eight-frame annotation template in which every point
|
||||
starts as `ignore-unreviewed`; it is explicitly not ground truth.
|
||||
- [ ] Complete independent human review for ground, curb, low obstacle,
|
||||
reflection noise and other non-ground points.
|
||||
- [ ] Measure accepted ground IoU, curb/low-obstacle recall and
|
||||
reflection-noise rejection against that reviewed generation.
|
||||
|
||||
Exit: an evidence-backed decision to retain or reject Patchwork++.
|
||||
The real diagnostic run is
|
||||
`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`.
|
||||
It covers all 66 frames and 226,963 points. The current local-percentile
|
||||
proposal classified 18.31% ground at p50 with 9.74 ms p95 host latency.
|
||||
Patchwork++ classified only 0.53% ground at p50 with 0.26 ms p95 host latency.
|
||||
Their point-aligned ground IoU was 2.90% p50 and disagreement was 17.92% p50.
|
||||
These last two values compare algorithms; they are not accuracy metrics.
|
||||
|
||||
The result is an evidence-backed **do-not-promote** decision for the current K1
|
||||
feed. Patchwork++ is fast, but its input model assumes a sensor-centric scan
|
||||
and physical sensor height. K1 `lio_pcl` is a vendor-mapped increment, its
|
||||
physical sensor height is not encoded by the best-effort pose, and scan
|
||||
geometry remains unknown. Translating the cloud until Patchwork++ looks
|
||||
plausible would tune against the candidate and invalidate the comparison.
|
||||
|
||||
The independent-label exit remains open. It can be closed by reviewing the
|
||||
content-bound template
|
||||
`ground-annotation-template-12e12eab0756c14f5e06adcaf13189e93188df9bb6d31224cb9b7d566ec15b18`,
|
||||
or by acquiring an admitted raw sensor scan with known physical sensor height
|
||||
and then producing a new benchmark generation.
|
||||
|
||||
### L3 — LiDAR-native 3D detection
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
# ADR 0020: do not promote Patchwork++ on vendor-mapped LiDAR increments
|
||||
|
||||
Date: 2026-07-25
|
||||
Status: accepted and implemented as a diagnostic gate
|
||||
|
||||
## Context
|
||||
|
||||
The L2 roadmap selected Patchwork++ as the first non-neural ground segmentation
|
||||
baseline. The official implementation is designed around a sensor-centric
|
||||
LiDAR scan, radial zones and a physical sensor height. Current K1 `lio_pcl`
|
||||
evidence instead contains a vendor-mapped point increment in `map`, an
|
||||
independent best-effort pose and no admitted raw sweep or scan geometry. The
|
||||
pose describes vendor odometry; it does not prove the physical height of the
|
||||
LiDAR above terrain.
|
||||
|
||||
Treating a successful Patchwork++ call as a valid baseline would conflate API
|
||||
compatibility with input-domain compatibility. Choosing a synthetic Z
|
||||
translation until the output looks plausible would use the candidate itself to
|
||||
define the normalization.
|
||||
|
||||
## Decision
|
||||
|
||||
1. Pin the official Patchwork++ v1.4.1 source at commit
|
||||
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`.
|
||||
2. Run it only behind the provider-neutral `lidar-ground/v1` diagnostic
|
||||
boundary.
|
||||
3. Bind the exact source commit and compiled binary SHA-256 into every result.
|
||||
4. Preserve point-aligned ground and assigned masks separately from raw replay.
|
||||
5. Compare it with the current local-percentile proposal using
|
||||
algorithm-to-algorithm IoU, disagreement and latency.
|
||||
6. Mark input-domain acceptance, labeled accuracy and production promotion
|
||||
false for current K1 vendor-map evidence.
|
||||
7. Never present algorithm IoU as ground IoU.
|
||||
8. Create an all-ignore, content-bound annotation template; only a separate
|
||||
human-reviewed generation may unlock ground IoU, curb/low-obstacle recall
|
||||
and reflection-noise rejection.
|
||||
9. Keep command, navigation and safety authority false.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Patchwork++ remains a useful candidate for a future raw vehicle LiDAR feed.
|
||||
- Its sub-millisecond host latency on the current slice does not compensate for
|
||||
the unaccepted input domain.
|
||||
- Current K1 work should prioritize independent labels and source evidence, not
|
||||
threshold tuning around a mis-specified sensor model.
|
||||
- The same immutable benchmark/API/React surface can compare a future raw scan,
|
||||
replay or simulation provider without moving C++ processing into React.
|
||||
|
||||
## Real diagnostic evidence
|
||||
|
||||
Benchmark
|
||||
`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`
|
||||
processed 66 frames and 226,963 points:
|
||||
|
||||
| Measurement | Current local percentile | Patchwork++ |
|
||||
| --- | ---: | ---: |
|
||||
| Ground fraction p50 | 18.31% | 0.53% |
|
||||
| Host latency p95 | 9.74 ms | 0.26 ms |
|
||||
|
||||
Algorithm ground IoU was 2.90% p50 and point disagreement was 17.92% p50.
|
||||
Neither is an accuracy metric. Independent labels are still missing.
|
||||
|
||||
## References
|
||||
|
||||
- `src/k1link/compute/lidar_ground.py`
|
||||
- `experiments/perception/run_lidar_ground_benchmark.py`
|
||||
- `src/k1link/web/lidar_api.py`
|
||||
- `apps/control-station/src/workspaces/LidarQualityWorkspace.tsx`
|
||||
- `docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md`
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute import (
|
||||
PATCHWORKPP_SOURCE_COMMIT,
|
||||
PATCHWORKPP_SOURCE_TAG,
|
||||
LidarGroundBenchmarkV1,
|
||||
LidarReplayPackV2,
|
||||
PatchworkPPGroundSegmenter,
|
||||
build_lidar_ground_annotation_template,
|
||||
build_lidar_ground_benchmark,
|
||||
)
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Run diagnostic-only local-percentile vs Patchwork++ ground A/B "
|
||||
"against one immutable LiDAR replay pack."
|
||||
)
|
||||
)
|
||||
parser.add_argument("replay_pack", type=Path)
|
||||
parser.add_argument("output_root", type=Path)
|
||||
parser.add_argument(
|
||||
"--annotation-root",
|
||||
type=Path,
|
||||
help="Optional root for an all-ignore human-review template.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--annotation-frames",
|
||||
type=int,
|
||||
default=8,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patchwork-module",
|
||||
default="pypatchworkpp",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patchwork-source-tag",
|
||||
default=PATCHWORKPP_SOURCE_TAG,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patchwork-source-commit",
|
||||
default=PATCHWORKPP_SOURCE_COMMIT,
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
arguments = _arguments()
|
||||
replay = LidarReplayPackV2(arguments.replay_pack)
|
||||
try:
|
||||
patchwork = PatchworkPPGroundSegmenter.load(
|
||||
module_name=arguments.patchwork_module,
|
||||
source_tag=arguments.patchwork_source_tag,
|
||||
source_commit=arguments.patchwork_source_commit,
|
||||
)
|
||||
output = build_lidar_ground_benchmark(
|
||||
replay,
|
||||
arguments.output_root,
|
||||
patchwork=patchwork,
|
||||
)
|
||||
annotation = (
|
||||
build_lidar_ground_annotation_template(
|
||||
replay,
|
||||
arguments.annotation_root,
|
||||
requested_frames=arguments.annotation_frames,
|
||||
)
|
||||
if arguments.annotation_root is not None
|
||||
else None
|
||||
)
|
||||
finally:
|
||||
replay.close()
|
||||
|
||||
result = LidarGroundBenchmarkV1(output)
|
||||
try:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"benchmark_id": result.benchmark_id,
|
||||
"status": result.report["status"],
|
||||
"decision": result.report["decision"],
|
||||
"current": result.report["current"],
|
||||
"candidate": result.report["candidate"],
|
||||
"comparison": result.report["comparison"],
|
||||
"annotation_template_id": (
|
||||
annotation.name if annotation is not None else None
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
result.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -69,6 +69,25 @@ from .lidar_contract import (
|
|||
lidar_readiness_document,
|
||||
sensor_frame_xyzi,
|
||||
)
|
||||
from .lidar_ground import (
|
||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA,
|
||||
LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA,
|
||||
LIDAR_GROUND_BENCHMARK_SCHEMA,
|
||||
PATCHWORKPP_SOURCE_COMMIT,
|
||||
PATCHWORKPP_SOURCE_TAG,
|
||||
PATCHWORKPP_SOURCE_URL,
|
||||
GroundBenchmarkProfile,
|
||||
GroundSegmentation,
|
||||
LidarGroundBenchmarkV1,
|
||||
LidarGroundError,
|
||||
LocalPercentileGroundSegmenter,
|
||||
PatchworkPPGroundSegmenter,
|
||||
build_lidar_ground_annotation_template,
|
||||
build_lidar_ground_benchmark,
|
||||
lidar_ground_benchmark_catalog_item,
|
||||
score_ground_labels,
|
||||
)
|
||||
from .lidar_replay import (
|
||||
LIDAR_EQUIVALENCE_REPORT_SCHEMA,
|
||||
LIDAR_QUALITY_REPORT_SCHEMA,
|
||||
|
|
@ -151,7 +170,12 @@ __all__ = [
|
|||
"EVALUATION_PACK_SCHEMA",
|
||||
"EvaluationFrameRequest",
|
||||
"EvaluationPackFrame",
|
||||
"GroundBenchmarkProfile",
|
||||
"GroundSegmentation",
|
||||
"LatestWinsQueue",
|
||||
"LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA",
|
||||
"LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA",
|
||||
"LIDAR_GROUND_BENCHMARK_SCHEMA",
|
||||
"LIDAR_EVIDENCE_PROFILE_SCHEMA",
|
||||
"LIDAR_EQUIVALENCE_REPORT_SCHEMA",
|
||||
"LIDAR_QUALITY_REPORT_SCHEMA",
|
||||
|
|
@ -169,6 +193,8 @@ __all__ = [
|
|||
"LidarPoseStatus",
|
||||
"LidarQualityMonitor",
|
||||
"LidarReadiness",
|
||||
"LidarGroundBenchmarkV1",
|
||||
"LidarGroundError",
|
||||
"LidarReplayError",
|
||||
"LidarReplayPackV2",
|
||||
"LidarReplayPointFrame",
|
||||
|
|
@ -190,6 +216,12 @@ __all__ = [
|
|||
"K1_LAB_LIDAR_PACK_V1_PROFILE",
|
||||
"K1_LIDAR_PACK_V2_PROFILE",
|
||||
"K1_LIVE_LIDAR_PROFILE",
|
||||
"DEFAULT_GROUND_BENCHMARK_PROFILE",
|
||||
"LocalPercentileGroundSegmenter",
|
||||
"PATCHWORKPP_SOURCE_COMMIT",
|
||||
"PATCHWORKPP_SOURCE_TAG",
|
||||
"PATCHWORKPP_SOURCE_URL",
|
||||
"PatchworkPPGroundSegmenter",
|
||||
"QueueSnapshot",
|
||||
"RecordedCalibratedFusion",
|
||||
"RecordedCalibratedFusionStore",
|
||||
|
|
@ -224,6 +256,8 @@ __all__ = [
|
|||
"prepare_recorded_qualification_slice",
|
||||
"assess_lidar_profile",
|
||||
"build_lidar_replay_pack_v2",
|
||||
"build_lidar_ground_annotation_template",
|
||||
"build_lidar_ground_benchmark",
|
||||
"DetectionFrame",
|
||||
"ObjectDetection",
|
||||
"RecordedPerceptionOverlayError",
|
||||
|
|
@ -246,6 +280,8 @@ __all__ = [
|
|||
"lidar_readiness_document",
|
||||
"lidar_pack_catalog_item",
|
||||
"lidar_pack_detail",
|
||||
"lidar_ground_benchmark_catalog_item",
|
||||
"score_ground_labels",
|
||||
"sensor_frame_xyzi",
|
||||
"verify_lidar_replay_equivalence",
|
||||
]
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -407,7 +407,14 @@ app.include_router(
|
|||
/ "compute-experiments"
|
||||
/ "lidar-replay-v2"
|
||||
/ "packs"
|
||||
)
|
||||
),
|
||||
ground_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "lidar-ground-v1"
|
||||
/ "benchmarks"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,14 +9,21 @@ from typing import Any, Final
|
|||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from k1link.compute import (
|
||||
LidarGroundBenchmarkV1,
|
||||
LidarGroundError,
|
||||
LidarReplayError,
|
||||
LidarReplayPackV2,
|
||||
lidar_ground_benchmark_catalog_item,
|
||||
lidar_pack_catalog_item,
|
||||
lidar_pack_detail,
|
||||
)
|
||||
|
||||
LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1"
|
||||
LIDAR_GROUND_CATALOG_SCHEMA: Final = (
|
||||
"missioncore.lidar-ground-benchmark-catalog/v1"
|
||||
)
|
||||
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
|
||||
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
|
||||
RootProvider = Callable[[], Path | None]
|
||||
|
||||
|
||||
|
|
@ -25,9 +32,15 @@ def configured_lidar_replay_root() -> Path | None:
|
|||
return Path(value).expanduser().absolute() if value else None
|
||||
|
||||
|
||||
def configured_lidar_ground_root() -> Path | None:
|
||||
value = os.environ.get("MISSIONCORE_LIDAR_GROUND_ROOT", "").strip()
|
||||
return Path(value).expanduser().absolute() if value else None
|
||||
|
||||
|
||||
def build_lidar_router(
|
||||
*,
|
||||
root_provider: RootProvider = configured_lidar_replay_root,
|
||||
ground_root_provider: RootProvider = configured_lidar_ground_root,
|
||||
) -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||
|
||||
|
|
@ -107,4 +120,95 @@ def build_lidar_router(
|
|||
detail="LiDAR replay pack не прошёл проверку целостности",
|
||||
) from exc
|
||||
|
||||
@router.get("/ground-benchmarks")
|
||||
def list_lidar_ground_benchmarks(
|
||||
pack_id: str | None = Query(default=None),
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
if pack_id is not None and _PACK_ID.fullmatch(pack_id) is None:
|
||||
raise HTTPException(status_code=404, detail="LiDAR replay pack не найден")
|
||||
root = ground_root_provider()
|
||||
if root is None or not root.is_dir():
|
||||
return {
|
||||
"schema_version": LIDAR_GROUND_CATALOG_SCHEMA,
|
||||
"configured": root is not None,
|
||||
"items": [],
|
||||
"valid_total": 0,
|
||||
"invalid_total": 0,
|
||||
"access": "read-only",
|
||||
}
|
||||
items: list[dict[str, object]] = []
|
||||
invalid_total = 0
|
||||
candidates = sorted(
|
||||
(
|
||||
candidate
|
||||
for candidate in root.iterdir()
|
||||
if candidate.is_dir()
|
||||
and _BENCHMARK_ID.fullmatch(candidate.name) is not None
|
||||
),
|
||||
key=lambda candidate: candidate.stat().st_mtime_ns,
|
||||
reverse=True,
|
||||
)
|
||||
for candidate in candidates:
|
||||
try:
|
||||
benchmark = LidarGroundBenchmarkV1(candidate)
|
||||
try:
|
||||
if (
|
||||
pack_id is None
|
||||
or benchmark.identity.get("replay_pack_id") == pack_id
|
||||
):
|
||||
items.append(
|
||||
lidar_ground_benchmark_catalog_item(benchmark)
|
||||
)
|
||||
finally:
|
||||
benchmark.close()
|
||||
except (LidarGroundError, OSError):
|
||||
invalid_total += 1
|
||||
return {
|
||||
"schema_version": LIDAR_GROUND_CATALOG_SCHEMA,
|
||||
"configured": True,
|
||||
"items": items[:limit],
|
||||
"valid_total": len(items),
|
||||
"invalid_total": invalid_total,
|
||||
"access": "read-only",
|
||||
}
|
||||
|
||||
@router.get("/ground-benchmarks/{benchmark_id}")
|
||||
def get_lidar_ground_benchmark(benchmark_id: str) -> dict[str, object]:
|
||||
if _BENCHMARK_ID.fullmatch(benchmark_id) is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="LiDAR ground benchmark не найден",
|
||||
)
|
||||
root = ground_root_provider()
|
||||
if root is None or not root.is_dir():
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="LiDAR ground storage не настроен",
|
||||
)
|
||||
candidate = root / benchmark_id
|
||||
if not candidate.is_dir():
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="LiDAR ground benchmark не найден",
|
||||
)
|
||||
try:
|
||||
benchmark = LidarGroundBenchmarkV1(candidate)
|
||||
try:
|
||||
return {
|
||||
"schema_version": (
|
||||
"missioncore.lidar-ground-benchmark-detail/v1"
|
||||
),
|
||||
"benchmark": lidar_ground_benchmark_catalog_item(benchmark),
|
||||
"report": benchmark.report,
|
||||
"access": "read-only",
|
||||
}
|
||||
finally:
|
||||
benchmark.close()
|
||||
except (LidarGroundError, OSError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="LiDAR ground benchmark не прошёл проверку целостности",
|
||||
) from exc
|
||||
|
||||
return router
|
||||
|
|
|
|||
|
|
@ -0,0 +1,306 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi import APIRouter
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from k1link.compute import (
|
||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
K1_LIDAR_PACK_V2_PROFILE,
|
||||
GroundSegmentation,
|
||||
LidarGroundBenchmarkV1,
|
||||
LidarGroundError,
|
||||
LidarReplayPointFrame,
|
||||
LidarReplayPoseFrame,
|
||||
LocalPercentileGroundSegmenter,
|
||||
PatchworkPPGroundSegmenter,
|
||||
build_lidar_ground_annotation_template,
|
||||
build_lidar_ground_benchmark,
|
||||
score_ground_labels,
|
||||
)
|
||||
from k1link.web.lidar_api import build_lidar_router
|
||||
|
||||
|
||||
class _Replay:
|
||||
def __init__(self) -> None:
|
||||
self.pack_id = f"lidar-replay-pack-{'a' * 64}"
|
||||
self.identity = {
|
||||
"logical_content_sha256": "b" * 64,
|
||||
"session_id": "synthetic-ground-session",
|
||||
}
|
||||
self.profile = K1_LIDAR_PACK_V2_PROFILE
|
||||
self._points = (
|
||||
np.asarray(
|
||||
[
|
||||
[-1.0, -1.0, 0.00],
|
||||
[-0.5, -1.0, 0.02],
|
||||
[0.0, -1.0, 0.01],
|
||||
[0.5, -1.0, 0.03],
|
||||
[1.0, -1.0, 0.00],
|
||||
[-1.0, 0.0, 0.01],
|
||||
[-0.5, 0.0, 0.02],
|
||||
[0.0, 0.0, 0.04],
|
||||
[0.5, 0.0, 0.35],
|
||||
[1.0, 0.0, 0.70],
|
||||
],
|
||||
dtype=np.float64,
|
||||
),
|
||||
np.asarray(
|
||||
[
|
||||
[-1.0, -1.0, 0.01],
|
||||
[-0.5, -1.0, 0.01],
|
||||
[0.0, -1.0, 0.02],
|
||||
[0.5, -1.0, 0.02],
|
||||
[1.0, -1.0, 0.01],
|
||||
[-1.0, 0.0, 0.02],
|
||||
[-0.5, 0.0, 0.01],
|
||||
[0.0, 0.0, 0.03],
|
||||
[0.5, 0.0, 0.40],
|
||||
[1.0, 0.0, 0.80],
|
||||
],
|
||||
dtype=np.float64,
|
||||
),
|
||||
)
|
||||
self.arrays = {
|
||||
"point_offsets": np.asarray([0, 10, 20], dtype="<i8"),
|
||||
"point_capture_sequence": np.asarray([1, 3], dtype="<i8"),
|
||||
"pose_received_monotonic_ns": np.asarray(
|
||||
[1_001_000_000, 1_101_000_000],
|
||||
dtype="<i8",
|
||||
),
|
||||
}
|
||||
|
||||
@property
|
||||
def point_frame_count(self) -> int:
|
||||
return 2
|
||||
|
||||
@property
|
||||
def pose_frame_count(self) -> int:
|
||||
return 2
|
||||
|
||||
@property
|
||||
def point_count(self) -> int:
|
||||
return 20
|
||||
|
||||
def point_frame(self, index: int) -> LidarReplayPointFrame:
|
||||
xyz = self._points[index]
|
||||
return LidarReplayPointFrame(
|
||||
capture_sequence=1 + index * 2,
|
||||
payload_bytes=100,
|
||||
received_at_epoch_ns=2_000_000_000 + index * 100_000_000,
|
||||
received_monotonic_ns=1_000_000_000 + index * 100_000_000,
|
||||
header_seq=10 + index,
|
||||
header_stamp=100 + index,
|
||||
scaler=1000,
|
||||
raw_xyz=(xyz * 1000).astype(np.int64),
|
||||
xyz_map=xyz,
|
||||
rgbi=np.full(10, 0xFFFFFF80, dtype=np.uint32),
|
||||
intensity=np.full(10, 128, dtype=np.uint8),
|
||||
)
|
||||
|
||||
def pose_frame(self, index: int) -> LidarReplayPoseFrame:
|
||||
return LidarReplayPoseFrame(
|
||||
capture_sequence=2 + index * 2,
|
||||
payload_bytes=80,
|
||||
received_at_epoch_ns=2_001_000_000 + index * 100_000_000,
|
||||
received_monotonic_ns=1_001_000_000 + index * 100_000_000,
|
||||
header_seq=20 + index,
|
||||
header_stamp=200 + index,
|
||||
header_scaler=1000,
|
||||
pose_stamp=201 + index,
|
||||
position_map=(0.0, 0.0, 0.0),
|
||||
orientation_map_from_lidar=(0.0, 0.0, 0.0, 1.0),
|
||||
distance=0.0,
|
||||
pose_accuracy=0.001,
|
||||
)
|
||||
|
||||
|
||||
class _Candidate:
|
||||
@property
|
||||
def identity(self) -> dict[str, object]:
|
||||
return {
|
||||
"provider_id": "test-patchwork/v1",
|
||||
"binary_sha256": "c" * 64,
|
||||
}
|
||||
|
||||
def segment(self, xyzi: np.ndarray) -> GroundSegmentation:
|
||||
ground = xyzi[:, 2] <= 0.025
|
||||
assigned = np.ones(xyzi.shape[0], dtype=np.bool_)
|
||||
return GroundSegmentation(ground, assigned, 0.5)
|
||||
|
||||
|
||||
def _endpoint(router: APIRouter, path: str) -> object:
|
||||
for route in router.routes:
|
||||
if isinstance(route, APIRoute) and route.path == path and "GET" in route.methods:
|
||||
return route.endpoint
|
||||
raise AssertionError(f"GET {path} route is missing")
|
||||
|
||||
|
||||
def test_local_percentile_ground_is_point_aligned_and_non_mutating() -> None:
|
||||
replay = _Replay()
|
||||
xyz = replay._points[0]
|
||||
xyzi = np.column_stack((xyz, np.ones(xyz.shape[0]))).astype(np.float32)
|
||||
unchanged = xyzi.copy()
|
||||
|
||||
result = LocalPercentileGroundSegmenter(
|
||||
profile=DEFAULT_GROUND_BENCHMARK_PROFILE
|
||||
).segment(xyzi)
|
||||
|
||||
assert result.ground_mask.shape == (10,)
|
||||
assert result.assigned_mask.all()
|
||||
assert 0 < np.count_nonzero(result.ground_mask) < 10
|
||||
np.testing.assert_array_equal(xyzi, unchanged)
|
||||
|
||||
|
||||
def test_ground_benchmark_is_immutable_diagnostic_evidence(tmp_path: Path) -> None:
|
||||
replay = _Replay()
|
||||
output = build_lidar_ground_benchmark(
|
||||
replay, # type: ignore[arg-type]
|
||||
tmp_path / "benchmarks",
|
||||
patchwork=_Candidate(),
|
||||
)
|
||||
result = LidarGroundBenchmarkV1(output)
|
||||
try:
|
||||
assert result.report["status"] == "diagnostic-only"
|
||||
assert result.report["input_domain"]["accepted"] is False
|
||||
assert result.report["labels"]["metrics_available"] is False
|
||||
assert (
|
||||
result.report["decision"]["status"]
|
||||
== "do-not-promote-on-current-vendor-map"
|
||||
)
|
||||
assert result.arrays["current_ground"].shape == (20,)
|
||||
assert result.arrays["candidate_ground"].shape == (20,)
|
||||
finally:
|
||||
result.close()
|
||||
|
||||
assert (
|
||||
build_lidar_ground_benchmark(
|
||||
replay, # type: ignore[arg-type]
|
||||
tmp_path / "benchmarks",
|
||||
patchwork=_Candidate(),
|
||||
)
|
||||
== output
|
||||
)
|
||||
|
||||
manifest_path = output / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
manifest["identity"]["points"] = 21
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
with pytest.raises(LidarGroundError, match="identity"):
|
||||
LidarGroundBenchmarkV1(output)
|
||||
|
||||
|
||||
def test_annotation_template_starts_all_ignore_and_never_ground_truth(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
replay = _Replay()
|
||||
output = build_lidar_ground_annotation_template(
|
||||
replay, # type: ignore[arg-type]
|
||||
tmp_path / "annotations",
|
||||
requested_frames=2,
|
||||
)
|
||||
manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8"))
|
||||
arrays = np.load(output / "labels-template.npz", allow_pickle=False)
|
||||
try:
|
||||
assert manifest["ground_truth"] is False
|
||||
assert manifest["identity"]["review_status"] == "unreviewed"
|
||||
assert arrays["labels"].shape == (20,)
|
||||
assert not np.any(arrays["labels"])
|
||||
finally:
|
||||
arrays.close()
|
||||
|
||||
|
||||
def test_ground_api_is_read_only_path_free_and_pack_filtered(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
replay = _Replay()
|
||||
root = tmp_path / "benchmarks"
|
||||
output = build_lidar_ground_benchmark(
|
||||
replay, # type: ignore[arg-type]
|
||||
root,
|
||||
patchwork=_Candidate(),
|
||||
)
|
||||
router = build_lidar_router(
|
||||
root_provider=lambda: None,
|
||||
ground_root_provider=lambda: root,
|
||||
)
|
||||
catalog_route = _endpoint(router, "/api/v1/lidar/ground-benchmarks")
|
||||
detail_route = _endpoint(
|
||||
router,
|
||||
"/api/v1/lidar/ground-benchmarks/{benchmark_id}",
|
||||
)
|
||||
|
||||
catalog = catalog_route( # type: ignore[operator]
|
||||
pack_id=replay.pack_id,
|
||||
limit=20,
|
||||
)
|
||||
detail = detail_route(benchmark_id=output.name) # type: ignore[operator]
|
||||
|
||||
assert (
|
||||
catalog["schema_version"]
|
||||
== "missioncore.lidar-ground-benchmark-catalog/v1"
|
||||
)
|
||||
assert catalog["valid_total"] == 1
|
||||
assert catalog["items"][0]["decision"]["production_promotion"] is False
|
||||
assert detail["benchmark"]["benchmark_id"] == output.name
|
||||
assert detail["access"] == "read-only"
|
||||
assert str(tmp_path) not in repr({"catalog": catalog, "detail": detail})
|
||||
|
||||
|
||||
def test_reviewed_ground_metrics_keep_ignore_out_of_denominators() -> None:
|
||||
prediction = GroundSegmentation(
|
||||
ground_mask=np.asarray([True, True, False, False, False, False]),
|
||||
assigned_mask=np.asarray([True, True, True, True, False, True]),
|
||||
latency_ms=1.0,
|
||||
)
|
||||
labels = np.asarray([1, 2, 2, 3, 4, 0], dtype=np.uint8)
|
||||
|
||||
metrics = score_ground_labels(prediction, labels)
|
||||
|
||||
assert metrics["reviewed_points"] == 5
|
||||
assert metrics["ground_iou"] == pytest.approx(0.5)
|
||||
assert metrics["curb_recall"] == pytest.approx(0.5)
|
||||
assert metrics["low_obstacle_recall"] == pytest.approx(1.0)
|
||||
assert metrics["reflection_noise_rejection"] == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_patchwork_adapter_records_binary_and_rejects_overlapping_indices(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
binary = tmp_path / "pypatchworkpp.so"
|
||||
binary.write_bytes(b"synthetic-binding")
|
||||
|
||||
class Parameters:
|
||||
pass
|
||||
|
||||
class Estimator:
|
||||
def __init__(self, _params: object) -> None:
|
||||
pass
|
||||
|
||||
def estimateGround(self, _points: np.ndarray) -> None:
|
||||
pass
|
||||
|
||||
def getGroundIndices(self) -> list[int]:
|
||||
return [0, 1]
|
||||
|
||||
def getNongroundIndices(self) -> list[int]:
|
||||
return [1, 2]
|
||||
|
||||
module = SimpleNamespace(
|
||||
__file__=str(binary),
|
||||
__version__="test",
|
||||
Parameters=Parameters,
|
||||
patchworkpp=Estimator,
|
||||
)
|
||||
adapter = PatchworkPPGroundSegmenter( # type: ignore[arg-type]
|
||||
module,
|
||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
)
|
||||
assert adapter.identity["binary_sha256"]
|
||||
with pytest.raises(LidarGroundError, match="assigned one point twice"):
|
||||
adapter.segment(np.zeros((3, 4), dtype=np.float32))
|
||||
Loading…
Reference in New Issue