feat(lidar): add ground segmentation diagnostic benchmark
This commit is contained in:
@@ -6,8 +6,10 @@ import { createServer } from "vite";
|
||||
let server;
|
||||
let parseLidarReplayCatalog;
|
||||
let parseLidarReplayDetail;
|
||||
let parseLidarGroundBenchmarkCatalog;
|
||||
let fetchLidarReplayCatalog;
|
||||
let fetchLidarReplayDetail;
|
||||
let fetchLidarGroundBenchmarks;
|
||||
let LidarReplayContractError;
|
||||
let workspaceById;
|
||||
|
||||
@@ -20,8 +22,10 @@ before(async () => {
|
||||
({
|
||||
parseLidarReplayCatalog,
|
||||
parseLidarReplayDetail,
|
||||
parseLidarGroundBenchmarkCatalog,
|
||||
fetchLidarReplayCatalog,
|
||||
fetchLidarReplayDetail,
|
||||
fetchLidarGroundBenchmarks,
|
||||
LidarReplayContractError,
|
||||
} = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts"));
|
||||
({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts"));
|
||||
@@ -130,6 +134,90 @@ function detail() {
|
||||
};
|
||||
}
|
||||
|
||||
function groundCatalog(overrides = {}) {
|
||||
const provider = (providerId, sourceCommit = undefined) => ({
|
||||
provider_id: providerId,
|
||||
source_commit: sourceCommit,
|
||||
});
|
||||
const branch = (providerValue, groundP50, latencyP95) => ({
|
||||
provider: providerValue,
|
||||
ground_fraction: {
|
||||
...distribution(),
|
||||
p50: groundP50,
|
||||
},
|
||||
assigned_fraction: {
|
||||
...distribution(),
|
||||
minimum: 1,
|
||||
mean: 1,
|
||||
p50: 1,
|
||||
p95: 1,
|
||||
maximum: 1,
|
||||
},
|
||||
latency_ms: {
|
||||
...distribution(),
|
||||
p95: latencyP95,
|
||||
},
|
||||
});
|
||||
return {
|
||||
schema_version: "missioncore.lidar-ground-benchmark-catalog/v1",
|
||||
configured: true,
|
||||
valid_total: 1,
|
||||
invalid_total: 0,
|
||||
access: "read-only",
|
||||
items: [{
|
||||
benchmark_id: `ground-benchmark-${"c".repeat(64)}`,
|
||||
replay_pack_id: packId,
|
||||
session_id: "20260719T220917Z_viewer_live",
|
||||
status: "diagnostic-only",
|
||||
frames: 66,
|
||||
points: 226963,
|
||||
input_domain: {
|
||||
accepted: false,
|
||||
representation: "vendor-map-increment",
|
||||
physical_sensor_height_known: false,
|
||||
sensor_scan_geometry_known: false,
|
||||
reason: "Patchwork++ expects sensor-centric scans.",
|
||||
},
|
||||
labels: {
|
||||
status: "missing-independent-review",
|
||||
metrics_available: false,
|
||||
},
|
||||
current: branch(
|
||||
provider("missioncore-local-percentile-ground/v1"),
|
||||
0.183,
|
||||
10.7,
|
||||
),
|
||||
candidate: branch(
|
||||
provider(
|
||||
"patchworkpp/v1.4.1",
|
||||
"3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||
),
|
||||
0.005,
|
||||
0.29,
|
||||
),
|
||||
comparison: {
|
||||
algorithm_to_algorithm_ground_iou: {
|
||||
...distribution(),
|
||||
p50: 0.029,
|
||||
},
|
||||
ground_disagreement_fraction: {
|
||||
...distribution(),
|
||||
p50: 0.179,
|
||||
},
|
||||
is_accuracy_metric: false,
|
||||
},
|
||||
decision: {
|
||||
status: "do-not-promote-on-current-vendor-map",
|
||||
production_promotion: false,
|
||||
reasons: ["candidate input domain is not accepted"],
|
||||
next_gate: "human-reviewed annotation subset",
|
||||
},
|
||||
created_at_utc: "2026-07-25T01:00:00Z",
|
||||
}],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function jsonResponse(payload, status = 200) {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
@@ -157,22 +245,48 @@ test("LiDAR contract refuses replay that did not pass equivalence", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("ground benchmark stays diagnostic until input and labels are accepted", () => {
|
||||
const parsed = parseLidarGroundBenchmarkCatalog(groundCatalog());
|
||||
|
||||
assert.equal(parsed.items[0].status, "diagnostic-only");
|
||||
assert.equal(parsed.items[0].inputDomain.accepted, false);
|
||||
assert.equal(parsed.items[0].labels.metricsAvailable, false);
|
||||
assert.equal(parsed.items[0].candidate.groundFraction.p50, 0.005);
|
||||
assert.equal(parsed.items[0].decision.productionPromotion, false);
|
||||
|
||||
const promoted = groundCatalog();
|
||||
promoted.items[0].decision.production_promotion = true;
|
||||
assert.throws(
|
||||
() => parseLidarGroundBenchmarkCatalog(promoted),
|
||||
LidarReplayContractError,
|
||||
);
|
||||
});
|
||||
|
||||
test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => {
|
||||
const calls = [];
|
||||
const fetcher = async (input, init) => {
|
||||
calls.push({ input: String(input), method: init?.method });
|
||||
if (String(input).includes("ground-benchmarks")) {
|
||||
return jsonResponse(groundCatalog());
|
||||
}
|
||||
return String(input).includes(packId)
|
||||
? jsonResponse(detail())
|
||||
: jsonResponse(catalog());
|
||||
};
|
||||
const parsedCatalog = await fetchLidarReplayCatalog({ fetcher });
|
||||
const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher });
|
||||
const ground = await fetchLidarGroundBenchmarks(packId, { fetcher });
|
||||
|
||||
assert.equal(parsedCatalog.validTotal, 1);
|
||||
assert.equal(parsedDetail.pack.packId, packId);
|
||||
assert.equal(ground.validTotal, 1);
|
||||
assert.deepEqual(calls, [
|
||||
{ input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" },
|
||||
{ input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" },
|
||||
{
|
||||
input: `/api/v1/lidar/ground-benchmarks?pack_id=${packId}&limit=20`,
|
||||
method: "GET",
|
||||
},
|
||||
]);
|
||||
assert.equal(workspaceById("lidar-quality").root, "data");
|
||||
assert.equal(workspaceById("lidar-quality").kind, "lidar-quality");
|
||||
|
||||
Reference in New Issue
Block a user