feat(lidar): admit and benchmark GOOSE baseline
This commit is contained in:
@@ -5,7 +5,11 @@ import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let parseDatasetGatewayCatalog;
|
||||
let parseDatasetNativeScanPreview;
|
||||
let parseDatasetGroundComparison;
|
||||
let fetchDatasetGatewayCatalog;
|
||||
let fetchDatasetNativeScanPreview;
|
||||
let fetchDatasetGroundComparison;
|
||||
let DatasetGatewayContractError;
|
||||
|
||||
before(async () => {
|
||||
@@ -16,7 +20,11 @@ before(async () => {
|
||||
});
|
||||
({
|
||||
parseDatasetGatewayCatalog,
|
||||
parseDatasetNativeScanPreview,
|
||||
parseDatasetGroundComparison,
|
||||
fetchDatasetGatewayCatalog,
|
||||
fetchDatasetNativeScanPreview,
|
||||
fetchDatasetGroundComparison,
|
||||
DatasetGatewayContractError,
|
||||
} = await server.ssrLoadModule("/src/core/lidar/datasetGateway.ts"));
|
||||
});
|
||||
@@ -27,7 +35,7 @@ after(async () => {
|
||||
|
||||
function catalog(overrides = {}) {
|
||||
return {
|
||||
schema_version: "missioncore.dataset-gateway-catalog/v1",
|
||||
schema_version: "missioncore.dataset-gateway-catalog/v2",
|
||||
access: "read-only",
|
||||
storage: {
|
||||
configured: false,
|
||||
@@ -35,6 +43,8 @@ function catalog(overrides = {}) {
|
||||
required_wsl_root: "/mnt/d/NDC_MISSIONCORE/datasets",
|
||||
admitted: false,
|
||||
status: "blocked-storage-policy",
|
||||
attestation: "none",
|
||||
manifest_valid: false,
|
||||
path_exposed: false,
|
||||
},
|
||||
sources: [{
|
||||
@@ -54,6 +64,8 @@ function catalog(overrides = {}) {
|
||||
},
|
||||
admission: {
|
||||
status: "blocked-storage-policy",
|
||||
archive: null,
|
||||
frame: null,
|
||||
},
|
||||
}],
|
||||
representations: [
|
||||
@@ -137,3 +149,99 @@ test("fetches the read-only gateway endpoint", async () => {
|
||||
assert.equal(calls[0].init.method, "GET");
|
||||
assert.equal(parsed.source.displayName, "GOOSE 3D");
|
||||
});
|
||||
|
||||
test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||
const payload = {
|
||||
schema_version: "missioncore.dataset-native-scan-preview/v1",
|
||||
source_id: "goose-3d/v2025-08-22",
|
||||
frame_id: "frame-1",
|
||||
representation: "native-scan",
|
||||
sampling: "deterministic-even-index",
|
||||
source_point_count: 2,
|
||||
point_count: 2,
|
||||
points_xyz_m: [[1, 2, 3], [4, 5, 6]],
|
||||
remission_0_to_255: [0, 255],
|
||||
semantic_label_ids: [23, 38],
|
||||
semantic_rgb_0_to_255: [255, 47, 128, 1, 51, 73],
|
||||
ground_truth_ground: [1, 0],
|
||||
classes: [
|
||||
{
|
||||
label_id: 23,
|
||||
class_name: "asphalt",
|
||||
hex: "#ff2f80",
|
||||
challenge_category_id: 2,
|
||||
challenge_category_name: "artificial_ground",
|
||||
},
|
||||
],
|
||||
safety: {
|
||||
visualization_only: true,
|
||||
navigation_or_safety_accepted: false,
|
||||
},
|
||||
};
|
||||
const parsed = parseDatasetNativeScanPreview(payload);
|
||||
assert.equal(parsed.pointCount, 2);
|
||||
assert.deepEqual(parsed.groundTruthGround, [1, 0]);
|
||||
|
||||
const fetched = await fetchDatasetNativeScanPreview({
|
||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
||||
});
|
||||
assert.equal(fetched.frameId, "frame-1");
|
||||
|
||||
payload.semantic_rgb_0_to_255.pop();
|
||||
assert.throws(
|
||||
() => parseDatasetNativeScanPreview(payload),
|
||||
DatasetGatewayContractError,
|
||||
);
|
||||
});
|
||||
|
||||
test("decodes a point-aligned current-vs-ground-truth comparison", async () => {
|
||||
const payload = {
|
||||
schema_version: "missioncore.dataset-ground-comparison-preview/v1",
|
||||
source_id: "goose-3d/v2025-08-22",
|
||||
frame_id: "frame-1",
|
||||
sampling: "deterministic-even-index",
|
||||
point_count: 2,
|
||||
current_ground: [1, 1],
|
||||
ground_truth_ground: [1, 0],
|
||||
evaluated: [1, 1],
|
||||
disagreement: [0, 1],
|
||||
metrics: {
|
||||
true_positive: 1,
|
||||
false_positive: 1,
|
||||
false_negative: 0,
|
||||
true_negative: 0,
|
||||
precision: 0.5,
|
||||
recall: 1,
|
||||
f1: 2 / 3,
|
||||
ground_iou: 0.5,
|
||||
accuracy: 0.5,
|
||||
artificial_ground_recall: 1,
|
||||
natural_ground_recall: 0,
|
||||
obstacle_non_ground_recall: 1,
|
||||
},
|
||||
latency_ms: 12.5,
|
||||
provider: {
|
||||
provider_id: "missioncore-local-percentile-ground/v1",
|
||||
implementation_sha256: "a".repeat(64),
|
||||
ground_truth: false,
|
||||
},
|
||||
safety: {
|
||||
qualification_only: true,
|
||||
navigation_or_safety_accepted: false,
|
||||
},
|
||||
};
|
||||
const parsed = parseDatasetGroundComparison(payload);
|
||||
assert.equal(parsed.metrics.groundIou, 0.5);
|
||||
assert.deepEqual(parsed.disagreement, [0, 1]);
|
||||
|
||||
const fetched = await fetchDatasetGroundComparison({
|
||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
||||
});
|
||||
assert.equal(fetched.latencyMs, 12.5);
|
||||
|
||||
payload.disagreement.pop();
|
||||
assert.throws(
|
||||
() => parseDatasetGroundComparison(payload),
|
||||
DatasetGatewayContractError,
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user