SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(lidar): add ground segmentation diagnostic benchmark

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-25 02:09:27 +03:00
parent e4fdd9fa20
commit 75a3e669d9
14 changed files with 2235 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/core/lidar/replayQuality.ts
+212
View File
@@ -59,6 +59,62 @@ export interface LidarReplayDetail {
stages: LidarStageReadiness[];
}
export interface LidarGroundProviderSummary {
providerId: string;
sourceCommit: string | null;
}
export interface LidarGroundBenchmark {
benchmarkId: string;
replayPackId: string;
sessionId: string;
status: "diagnostic-only";
frames: number;
points: number;
inputDomain: {
accepted: false;
representation: string;
physicalSensorHeightKnown: boolean;
sensorScanGeometryKnown: boolean;
reason: string;
};
labels: {
status: "missing-independent-review";
metricsAvailable: false;
};
current: {
provider: LidarGroundProviderSummary;
groundFraction: LidarDistribution;
assignedFraction: LidarDistribution;
latencyMs: LidarDistribution;
};
candidate: {
provider: LidarGroundProviderSummary;
groundFraction: LidarDistribution;
assignedFraction: LidarDistribution;
latencyMs: LidarDistribution;
};
comparison: {
algorithmGroundIou: LidarDistribution;
groundDisagreementFraction: LidarDistribution;
isAccuracyMetric: false;
};
decision: {
status: "do-not-promote-on-current-vendor-map";
productionPromotion: false;
reasons: string[];
nextGate: string;
};
createdAtUtc: string | null;
}
export interface LidarGroundBenchmarkCatalog {
configured: boolean;
validTotal: number;
invalidTotal: number;
items: LidarGroundBenchmark[];
}
export class LidarReplayContractError extends Error {}
export class LidarReplayApiError extends Error {
@@ -73,8 +129,10 @@ type LidarFetch = (
) => Promise<Response>;
const SAFE_PACK_ID = /^lidar-replay-pack-[a-f0-9]{64}$/;
const SAFE_GROUND_BENCHMARK_ID = /^ground-benchmark-[a-f0-9]{64}$/;
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$/;
const SHA256 = /^[a-f0-9]{64}$/;
const GIT_SHA1 = /^[a-f0-9]{40}$/;
function record(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
@@ -258,6 +316,139 @@ export function parseLidarReplayDetail(value: unknown): LidarReplayDetail {
};
}
function groundProvider(
value: unknown,
label: string,
): LidarGroundProviderSummary {
const source = record(value, label);
return {
providerId: string(source.provider_id, `${label}.provider_id`, SAFE_ID),
sourceCommit:
source.source_commit === undefined || source.source_commit === null
? null
: string(source.source_commit, `${label}.source_commit`, GIT_SHA1),
};
}
function groundBranch(
value: unknown,
label: string,
): LidarGroundBenchmark["current"] {
const source = record(value, label);
return {
provider: groundProvider(source.provider, `${label}.provider`),
groundFraction: distribution(
source.ground_fraction,
`${label}.ground_fraction`,
),
assignedFraction: distribution(
source.assigned_fraction,
`${label}.assigned_fraction`,
),
latencyMs: distribution(source.latency_ms, `${label}.latency_ms`),
};
}
function groundBenchmark(value: unknown): LidarGroundBenchmark {
const source = record(value, "LiDAR ground benchmark");
if (source.status !== "diagnostic-only") {
throw new LidarReplayContractError("Ground benchmark status несовместим");
}
const inputDomain = record(source.input_domain, "input_domain");
const labels = record(source.labels, "labels");
const comparison = record(source.comparison, "comparison");
const decision = record(source.decision, "decision");
if (
inputDomain.accepted !== false
|| labels.status !== "missing-independent-review"
|| labels.metrics_available !== false
|| comparison.is_accuracy_metric !== false
|| decision.status !== "do-not-promote-on-current-vendor-map"
|| decision.production_promotion !== false
) {
throw new LidarReplayContractError(
"Ground benchmark завышает readiness или accuracy",
);
}
return {
benchmarkId: string(
source.benchmark_id,
"benchmark_id",
SAFE_GROUND_BENCHMARK_ID,
),
replayPackId: string(source.replay_pack_id, "replay_pack_id", SAFE_PACK_ID),
sessionId: string(source.session_id, "session_id", SAFE_ID),
status: "diagnostic-only",
frames: integer(source.frames, "frames"),
points: integer(source.points, "points"),
inputDomain: {
accepted: false,
representation: string(
inputDomain.representation,
"input_domain.representation",
SAFE_ID,
),
physicalSensorHeightKnown: boolean(
inputDomain.physical_sensor_height_known,
"input_domain.physical_sensor_height_known",
),
sensorScanGeometryKnown: boolean(
inputDomain.sensor_scan_geometry_known,
"input_domain.sensor_scan_geometry_known",
),
reason: string(inputDomain.reason, "input_domain.reason"),
},
labels: {
status: "missing-independent-review",
metricsAvailable: false,
},
current: groundBranch(source.current, "current"),
candidate: groundBranch(source.candidate, "candidate"),
comparison: {
algorithmGroundIou: distribution(
comparison.algorithm_to_algorithm_ground_iou,
"comparison.algorithm_ground_iou",
),
groundDisagreementFraction: distribution(
comparison.ground_disagreement_fraction,
"comparison.ground_disagreement_fraction",
),
isAccuracyMetric: false,
},
decision: {
status: "do-not-promote-on-current-vendor-map",
productionPromotion: false,
reasons: array(decision.reasons, "decision.reasons").map((reason) =>
string(reason, "decision.reason")
),
nextGate: string(decision.next_gate, "decision.next_gate"),
},
createdAtUtc:
source.created_at_utc === null || source.created_at_utc === undefined
? null
: string(source.created_at_utc, "created_at_utc"),
};
}
export function parseLidarGroundBenchmarkCatalog(
value: unknown,
): LidarGroundBenchmarkCatalog {
const source = record(value, "LiDAR ground catalog");
if (
source.schema_version
!== "missioncore.lidar-ground-benchmark-catalog/v1"
|| source.access !== "read-only"
) {
throw new LidarReplayContractError("LiDAR ground catalog несовместим");
}
return {
configured: boolean(source.configured, "configured"),
validTotal: integer(source.valid_total, "valid_total"),
invalidTotal: integer(source.invalid_total, "invalid_total"),
items: array(source.items, "items").map(groundBenchmark),
};
}
async function responseJson(
response: Response,
fallback: string,
@@ -309,3 +500,24 @@ export async function fetchLidarReplayDetail(
await responseJson(response, "Не удалось получить LiDAR quality report."),
);
}
export async function fetchLidarGroundBenchmarks(
packId: string,
options: { signal?: AbortSignal; fetcher?: LidarFetch } = {},
): Promise<LidarGroundBenchmarkCatalog> {
if (!SAFE_PACK_ID.test(packId)) {
throw new LidarReplayContractError("Некорректный LiDAR pack id");
}
const fetcher = options.fetcher ?? fetch;
const response = await fetcher(
`/api/v1/lidar/ground-benchmarks?pack_id=${encodeURIComponent(packId)}&limit=20`,
{
method: "GET",
headers: { Accept: "application/json" },
signal: options.signal,
},
);
return parseLidarGroundBenchmarkCatalog(
await responseJson(response, "Не удалось получить LiDAR ground benchmark."),
);
}
apps/control-station/src/styles/responsive.css
+9
View File
@@ -22,6 +22,10 @@
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.lidar-ground-metrics {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.pipeline-strip {
grid-template-columns: repeat(4, minmax(0, 1fr));
}
@@ -162,6 +166,11 @@
grid-template-columns: 1fr;
}
.lidar-ground-metrics,
.lidar-ground-gates {
grid-template-columns: 1fr;
}
.polygon-run-identity dl {
grid-template-columns: 1fr;
}
apps/control-station/src/styles/workspaces.css
+70
View File
@@ -954,6 +954,76 @@
margin-top: 0.8rem;
}
.lidar-ground-benchmark {
min-width: 0;
}
.lidar-ground-metrics {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 0.65rem;
margin-top: 1rem;
}
.lidar-ground-metrics > div {
display: grid;
gap: 0.3rem;
border: 1px solid var(--station-hairline);
border-radius: 0.8rem;
background: rgb(255 255 255 / 0.025);
padding: 0.75rem;
}
.lidar-ground-metrics span,
.lidar-ground-metrics small,
.lidar-ground-gates p,
.lidar-ground-footer,
.lidar-ground-empty {
color: var(--nodedc-text-muted);
font-size: 0.62rem;
line-height: 1.45;
}
.lidar-ground-metrics strong {
color: var(--nodedc-text-primary);
font-size: 1.15rem;
font-weight: 650;
}
.lidar-ground-gates {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.65rem;
margin-top: 0.75rem;
}
.lidar-ground-gates > div {
display: grid;
align-content: start;
gap: 0.55rem;
border-left: 1px solid var(--station-hairline);
padding-left: 0.7rem;
}
.lidar-ground-gates p,
.lidar-ground-empty {
margin: 0;
}
.lidar-ground-footer {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
gap: 0.5rem;
margin-top: 0.85rem;
border-top: 1px solid var(--station-hairline);
padding-top: 0.7rem;
}
.lidar-ground-empty {
margin-top: 1rem;
}
.overview-grid {
display: grid;
grid-template-columns: minmax(0, 1.55fr) minmax(19rem, 0.75fr);
apps/control-station/src/workspaces/LidarQualityWorkspace.tsx
+127 -3
View File
@@ -6,8 +6,10 @@ import {
} from "@nodedc/ui-react";
import {
fetchLidarGroundBenchmarks,
fetchLidarReplayCatalog,
fetchLidarReplayDetail,
type LidarGroundBenchmark,
type LidarReplayCatalog,
type LidarReplayDetail,
type LidarStageReadiness,
@@ -53,6 +55,8 @@ export function LidarQualityWorkspace({
}) {
const [catalog, setCatalog] = useState<LidarReplayCatalog | null>(null);
const [detail, setDetail] = useState<LidarReplayDetail | null>(null);
const [groundBenchmark, setGroundBenchmark] =
useState<LidarGroundBenchmark | null>(null);
const [selectedPackId, setSelectedPackId] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
@@ -72,17 +76,25 @@ export function LidarQualityWorkspace({
const target = selectedPackId ?? nextCatalog.items[0]?.packId ?? null;
if (!target) {
setDetail(null);
setGroundBenchmark(null);
return;
}
const nextDetail = await fetchLidarReplayDetail(target, {
signal: controller.signal,
});
const [nextDetail, groundCatalog] = await Promise.all([
fetchLidarReplayDetail(target, {
signal: controller.signal,
}),
fetchLidarGroundBenchmarks(target, {
signal: controller.signal,
}),
]);
if (controller.signal.aborted) return;
setSelectedPackId(target);
setDetail(nextDetail);
setGroundBenchmark(groundCatalog.items[0] ?? null);
} catch (loadError) {
if (controller.signal.aborted) return;
setDetail(null);
setGroundBenchmark(null);
setError(errorMessage(loadError));
} finally {
if (!controller.signal.aborted) setLoading(false);
@@ -226,6 +238,118 @@ export function LidarQualityWorkspace({
</GlassSurface>
</div>
<GlassSurface className="lidar-ground-benchmark" padding="lg">
<header className="panel-heading">
<div>
<span className="section-eyebrow">GROUND A/B · L2</span>
<h2>Текущий heuristic против Patchwork++</h2>
</div>
<StatusBadge tone={groundBenchmark ? "warning" : "neutral"}>
{groundBenchmark ? "Diagnostic only" : "Нет результата"}
</StatusBadge>
</header>
{groundBenchmark ? (
<>
<section
className="lidar-ground-metrics"
aria-label="Ground segmentation A/B"
>
<div>
<span>Текущий ground p50</span>
<strong>
{formatFraction(
groundBenchmark.current.groundFraction.p50,
)}
</strong>
<small>
{formatNumber(
groundBenchmark.current.latencyMs.p95,
2,
)}{" "}
мс p95
</small>
</div>
<div>
<span>Patchwork++ ground p50</span>
<strong>
{formatFraction(
groundBenchmark.candidate.groundFraction.p50,
)}
</strong>
<small>
{formatNumber(
groundBenchmark.candidate.latencyMs.p95,
2,
)}{" "}
мс p95
</small>
</div>
<div>
<span>Algorithm IoU p50</span>
<strong>
{formatFraction(
groundBenchmark.comparison.algorithmGroundIou.p50,
)}
</strong>
<small>Не является accuracy</small>
</div>
<div>
<span>Disagreement p50</span>
<strong>
{formatFraction(
groundBenchmark.comparison
.groundDisagreementFraction.p50,
)}
</strong>
<small>
{groundBenchmark.frames.toLocaleString("ru-RU")} кадров
</small>
</div>
</section>
<div className="lidar-ground-gates">
<div>
<StatusBadge tone="danger">
Входной контракт не принят
</StatusBadge>
<p>
Patchwork++ ожидает sensor-centric scan и физическую высоту
сенсора; текущий point feed является vendor-mapped increment.
</p>
</div>
<div>
<StatusBadge tone="warning">Разметка не принята</StatusBadge>
<p>
IoU, curb recall, low-obstacle recall и reflection-noise
rejection появятся только после независимого human review.
</p>
</div>
<div>
<StatusBadge tone="danger">Не продвигать</StatusBadge>
<p>
Следующий gate: human-reviewed annotation subset или
принятый raw sensor scan с физической высотой сенсора.
</p>
</div>
</div>
<footer className="lidar-ground-footer">
<span>
{groundBenchmark.current.provider.providerId}
</span>
<span>
{groundBenchmark.candidate.provider.providerId}
{groundBenchmark.candidate.provider.sourceCommit
? ` · ${groundBenchmark.candidate.provider.sourceCommit.slice(0, 12)}`
: ""}
</span>
</footer>
</>
) : (
<p className="lidar-ground-empty">
Для выбранного replay pack ещё нет проверенного ground benchmark.
</p>
)}
</GlassSurface>
<GlassSurface className="lidar-pack-catalog" padding="lg">
<header className="panel-heading">
<div>
apps/control-station/test/lidarReplayQuality.test.mjs
+114
View File
@@ -6,8 +6,10 @@ import { createServer } from "vite";
let server;
let parseLidarReplayCatalog;
let parseLidarReplayDetail;
let parseLidarGroundBenchmarkCatalog;
let fetchLidarReplayCatalog;
let fetchLidarReplayDetail;
let fetchLidarGroundBenchmarks;
let LidarReplayContractError;
let workspaceById;
@@ -20,8 +22,10 @@ before(async () => {
({
parseLidarReplayCatalog,
parseLidarReplayDetail,
parseLidarGroundBenchmarkCatalog,
fetchLidarReplayCatalog,
fetchLidarReplayDetail,
fetchLidarGroundBenchmarks,
LidarReplayContractError,
} = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts"));
({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts"));
@@ -130,6 +134,90 @@ function detail() {
};
}
function groundCatalog(overrides = {}) {
const provider = (providerId, sourceCommit = undefined) => ({
provider_id: providerId,
source_commit: sourceCommit,
});
const branch = (providerValue, groundP50, latencyP95) => ({
provider: providerValue,
ground_fraction: {
...distribution(),
p50: groundP50,
},
assigned_fraction: {
...distribution(),
minimum: 1,
mean: 1,
p50: 1,
p95: 1,
maximum: 1,
},
latency_ms: {
...distribution(),
p95: latencyP95,
},
});
return {
schema_version: "missioncore.lidar-ground-benchmark-catalog/v1",
configured: true,
valid_total: 1,
invalid_total: 0,
access: "read-only",
items: [{
benchmark_id: `ground-benchmark-${"c".repeat(64)}`,
replay_pack_id: packId,
session_id: "20260719T220917Z_viewer_live",
status: "diagnostic-only",
frames: 66,
points: 226963,
input_domain: {
accepted: false,
representation: "vendor-map-increment",
physical_sensor_height_known: false,
sensor_scan_geometry_known: false,
reason: "Patchwork++ expects sensor-centric scans.",
},
labels: {
status: "missing-independent-review",
metrics_available: false,
},
current: branch(
provider("missioncore-local-percentile-ground/v1"),
0.183,
10.7,
),
candidate: branch(
provider(
"patchworkpp/v1.4.1",
"3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
),
0.005,
0.29,
),
comparison: {
algorithm_to_algorithm_ground_iou: {
...distribution(),
p50: 0.029,
},
ground_disagreement_fraction: {
...distribution(),
p50: 0.179,
},
is_accuracy_metric: false,
},
decision: {
status: "do-not-promote-on-current-vendor-map",
production_promotion: false,
reasons: ["candidate input domain is not accepted"],
next_gate: "human-reviewed annotation subset",
},
created_at_utc: "2026-07-25T01:00:00Z",
}],
...overrides,
};
}
function jsonResponse(payload, status = 200) {
return new Response(JSON.stringify(payload), {
status,
@@ -157,22 +245,48 @@ test("LiDAR contract refuses replay that did not pass equivalence", () => {
);
});
test("ground benchmark stays diagnostic until input and labels are accepted", () => {
const parsed = parseLidarGroundBenchmarkCatalog(groundCatalog());
assert.equal(parsed.items[0].status, "diagnostic-only");
assert.equal(parsed.items[0].inputDomain.accepted, false);
assert.equal(parsed.items[0].labels.metricsAvailable, false);
assert.equal(parsed.items[0].candidate.groundFraction.p50, 0.005);
assert.equal(parsed.items[0].decision.productionPromotion, false);
const promoted = groundCatalog();
promoted.items[0].decision.production_promotion = true;
assert.throws(
() => parseLidarGroundBenchmarkCatalog(promoted),
LidarReplayContractError,
);
});
test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => {
const calls = [];
const fetcher = async (input, init) => {
calls.push({ input: String(input), method: init?.method });
if (String(input).includes("ground-benchmarks")) {
return jsonResponse(groundCatalog());
}
return String(input).includes(packId)
? jsonResponse(detail())
: jsonResponse(catalog());
};
const parsedCatalog = await fetchLidarReplayCatalog({ fetcher });
const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher });
const ground = await fetchLidarGroundBenchmarks(packId, { fetcher });
assert.equal(parsedCatalog.validTotal, 1);
assert.equal(parsedDetail.pack.packId, packId);
assert.equal(ground.validTotal, 1);
assert.deepEqual(calls, [
{ input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" },
{ input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" },
{
input: `/api/v1/lidar/ground-benchmarks?pack_id=${packId}&limit=20`,
method: "GET",
},
]);
assert.equal(workspaceById("lidar-quality").root, "data");
assert.equal(workspaceById("lidar-quality").kind, "lidar-quality");