feat(lidar): qualify Patchwork++ on GOOSE
This commit is contained in:
parent
951b40c870
commit
60ba64004b
|
|
@ -90,15 +90,7 @@ export interface DatasetNativeScanPreview {
|
|||
}>;
|
||||
}
|
||||
|
||||
export interface DatasetGroundComparison {
|
||||
sourceId: "goose-3d/v2025-08-22";
|
||||
frameId: string;
|
||||
pointCount: number;
|
||||
currentGround: number[];
|
||||
groundTruthGround: number[];
|
||||
evaluated: number[];
|
||||
disagreement: number[];
|
||||
metrics: {
|
||||
export interface DatasetGroundMetrics {
|
||||
precision: number;
|
||||
recall: number;
|
||||
f1: number;
|
||||
|
|
@ -107,8 +99,34 @@ export interface DatasetGroundComparison {
|
|||
artificialGroundRecall: number;
|
||||
naturalGroundRecall: number;
|
||||
obstacleNonGroundRecall: number;
|
||||
}
|
||||
|
||||
export interface DatasetGroundComparison {
|
||||
sourceId: "goose-3d/v2025-08-22";
|
||||
frameId: string;
|
||||
pointCount: number;
|
||||
currentGround: number[];
|
||||
patchworkGround: number[];
|
||||
patchworkAssigned: number[];
|
||||
groundTruthGround: number[];
|
||||
evaluated: number[];
|
||||
currentDisagreement: number[];
|
||||
patchworkDisagreement: number[];
|
||||
metrics: {
|
||||
current: DatasetGroundMetrics;
|
||||
patchworkpp: DatasetGroundMetrics;
|
||||
};
|
||||
latencyMs: {
|
||||
current: number;
|
||||
patchworkpp: number;
|
||||
};
|
||||
patchworkAssignedFraction: number;
|
||||
inputProfile: {
|
||||
frameId: "sensor/lidar/vls128_roof";
|
||||
sensorHeightM: number;
|
||||
heightEvidence: "published-dimensioned-schematic";
|
||||
normalizedScanProduced: false;
|
||||
};
|
||||
latencyMs: number;
|
||||
}
|
||||
|
||||
export class DatasetGatewayContractError extends Error {}
|
||||
|
|
@ -500,7 +518,7 @@ export function parseDatasetGroundComparison(
|
|||
): DatasetGroundComparison {
|
||||
const source = record(value, "Ground comparison");
|
||||
if (
|
||||
source.schema_version !== "missioncore.dataset-ground-comparison-preview/v1"
|
||||
source.schema_version !== "missioncore.dataset-ground-comparison-preview/v2"
|
||||
|| source.source_id !== "goose-3d/v2025-08-22"
|
||||
|| source.sampling !== "deterministic-even-index"
|
||||
) {
|
||||
|
|
@ -508,41 +526,135 @@ export function parseDatasetGroundComparison(
|
|||
}
|
||||
const pointCount = number(source.point_count, "point_count");
|
||||
const currentGround = integers(source.current_ground, "current_ground", 1);
|
||||
const patchworkGround = integers(source.patchwork_ground, "patchwork_ground", 1);
|
||||
const patchworkAssigned = integers(
|
||||
source.patchwork_assigned,
|
||||
"patchwork_assigned",
|
||||
1,
|
||||
);
|
||||
const groundTruthGround = integers(
|
||||
source.ground_truth_ground,
|
||||
"ground_truth_ground",
|
||||
1,
|
||||
);
|
||||
const evaluated = integers(source.evaluated, "evaluated", 1);
|
||||
const disagreement = integers(source.disagreement, "disagreement", 1);
|
||||
const currentDisagreement = integers(
|
||||
source.current_disagreement,
|
||||
"current_disagreement",
|
||||
1,
|
||||
);
|
||||
const patchworkDisagreement = integers(
|
||||
source.patchwork_disagreement,
|
||||
"patchwork_disagreement",
|
||||
1,
|
||||
);
|
||||
if (
|
||||
pointCount < 1
|
||||
|| pointCount > 50_000
|
||||
|| currentGround.length !== pointCount
|
||||
|| patchworkGround.length !== pointCount
|
||||
|| patchworkAssigned.length !== pointCount
|
||||
|| groundTruthGround.length !== pointCount
|
||||
|| evaluated.length !== pointCount
|
||||
|| disagreement.length !== pointCount
|
||||
|| currentDisagreement.length !== pointCount
|
||||
|| patchworkDisagreement.length !== pointCount
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Ground comparison arrays не выровнены");
|
||||
}
|
||||
const metrics = record(source.metrics, "metrics");
|
||||
const parseMetrics = (
|
||||
value: unknown,
|
||||
label: string,
|
||||
): DatasetGroundMetrics => {
|
||||
const providerMetrics = record(value, label);
|
||||
const fraction = (key: string): number => {
|
||||
const value = number(metrics[key], `metrics.${key}`);
|
||||
if (value > 1) {
|
||||
throw new DatasetGatewayContractError(`metrics.${key}: ожидалась доля`);
|
||||
const result = number(providerMetrics[key], `${label}.${key}`);
|
||||
if (result > 1) {
|
||||
throw new DatasetGatewayContractError(`${label}.${key}: ожидалась доля`);
|
||||
}
|
||||
return value;
|
||||
return result;
|
||||
};
|
||||
const provider = record(source.provider, "provider");
|
||||
return {
|
||||
precision: fraction("precision"),
|
||||
recall: fraction("recall"),
|
||||
f1: fraction("f1"),
|
||||
groundIou: fraction("ground_iou"),
|
||||
accuracy: fraction("accuracy"),
|
||||
artificialGroundRecall: fraction("artificial_ground_recall"),
|
||||
naturalGroundRecall: fraction("natural_ground_recall"),
|
||||
obstacleNonGroundRecall: fraction("obstacle_non_ground_recall"),
|
||||
};
|
||||
};
|
||||
const currentMetrics = parseMetrics(metrics.current, "metrics.current");
|
||||
const patchworkMetrics = parseMetrics(
|
||||
metrics.patchworkpp,
|
||||
"metrics.patchworkpp",
|
||||
);
|
||||
const providers = record(source.providers, "providers");
|
||||
const currentProvider = record(providers.current, "providers.current");
|
||||
const patchworkProvider = record(providers.patchworkpp, "providers.patchworkpp");
|
||||
if (
|
||||
provider.provider_id !== "missioncore-local-percentile-ground/v1"
|
||||
|| provider.ground_truth !== false
|
||||
currentProvider.provider_id !== "missioncore-local-percentile-ground/v1"
|
||||
|| currentProvider.ground_truth !== false
|
||||
|| !/^[a-f0-9]{64}$/.test(
|
||||
string(provider.implementation_sha256, "provider.implementation_sha256"),
|
||||
string(
|
||||
currentProvider.implementation_sha256,
|
||||
"providers.current.implementation_sha256",
|
||||
),
|
||||
)
|
||||
|| patchworkProvider.provider_id !== "patchworkpp/v1.4.1"
|
||||
|| patchworkProvider.source_tag !== "v1.4.1"
|
||||
|| patchworkProvider.source_commit
|
||||
!== "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||
|| patchworkProvider.ground_truth !== false
|
||||
|| !/^[a-f0-9]{64}$/.test(
|
||||
string(
|
||||
patchworkProvider.binary_sha256,
|
||||
"providers.patchworkpp.binary_sha256",
|
||||
),
|
||||
)
|
||||
) {
|
||||
throw new DatasetGatewayContractError("Ground comparison provider несовместим");
|
||||
}
|
||||
const latency = record(source.latency_ms, "latency_ms");
|
||||
const inputProfile = record(source.input_profile, "input_profile");
|
||||
const sensorFrame = record(inputProfile.sensor_frame, "input_profile.sensor_frame");
|
||||
const height = record(inputProfile.height, "input_profile.height");
|
||||
const scope = record(inputProfile.scope, "input_profile.scope");
|
||||
const sensorHeightM = number(
|
||||
height.sensor_above_ground_m,
|
||||
"height.sensor_above_ground_m",
|
||||
);
|
||||
if (
|
||||
inputProfile.schema_version !== "missioncore.goose-patchwork-profile/v1"
|
||||
|| inputProfile.source_id !== "goose-3d/v2025-08-22"
|
||||
|| inputProfile.representation !== "native-scan"
|
||||
|| sensorFrame.frame_id !== "sensor/lidar/vls128_roof"
|
||||
|| sensorFrame.handedness !== "right"
|
||||
|| sensorFrame.x !== "forward"
|
||||
|| sensorFrame.y !== "left"
|
||||
|| sensorFrame.z !== "up"
|
||||
|| sensorFrame.one_revolution !== true
|
||||
|| height.base_link_above_ground_m !== 0.64
|
||||
|| height.lidar_above_base_link_m !== 1.6
|
||||
|| sensorHeightM !== 2.24
|
||||
|| height.evidence !== "published-dimensioned-schematic"
|
||||
|| scope.patchworkpp_eligible !== true
|
||||
|| scope.normalized_scan_produced !== false
|
||||
|| scope.complete_vehicle_transform_known !== false
|
||||
|| scope.deskew_claimed !== false
|
||||
) {
|
||||
throw new DatasetGatewayContractError("GOOSE Patchwork++ profile несовместим");
|
||||
}
|
||||
const patchworkAssignedFraction = number(
|
||||
source.patchwork_assigned_fraction,
|
||||
"patchwork_assigned_fraction",
|
||||
);
|
||||
if (patchworkAssignedFraction > 1) {
|
||||
throw new DatasetGatewayContractError(
|
||||
"patchwork_assigned_fraction: ожидалась доля",
|
||||
);
|
||||
}
|
||||
const safety = record(source.safety, "safety");
|
||||
if (
|
||||
safety.qualification_only !== true
|
||||
|
|
@ -555,20 +667,27 @@ export function parseDatasetGroundComparison(
|
|||
frameId: string(source.frame_id, "frame_id", true),
|
||||
pointCount,
|
||||
currentGround,
|
||||
patchworkGround,
|
||||
patchworkAssigned,
|
||||
groundTruthGround,
|
||||
evaluated,
|
||||
disagreement,
|
||||
currentDisagreement,
|
||||
patchworkDisagreement,
|
||||
metrics: {
|
||||
precision: fraction("precision"),
|
||||
recall: fraction("recall"),
|
||||
f1: fraction("f1"),
|
||||
groundIou: fraction("ground_iou"),
|
||||
accuracy: fraction("accuracy"),
|
||||
artificialGroundRecall: fraction("artificial_ground_recall"),
|
||||
naturalGroundRecall: fraction("natural_ground_recall"),
|
||||
obstacleNonGroundRecall: fraction("obstacle_non_ground_recall"),
|
||||
current: currentMetrics,
|
||||
patchworkpp: patchworkMetrics,
|
||||
},
|
||||
latencyMs: {
|
||||
current: number(latency.current, "latency_ms.current"),
|
||||
patchworkpp: number(latency.patchworkpp, "latency_ms.patchworkpp"),
|
||||
},
|
||||
patchworkAssignedFraction,
|
||||
inputProfile: {
|
||||
frameId: "sensor/lidar/vls128_roof",
|
||||
sensorHeightM,
|
||||
heightEvidence: "published-dimensioned-schematic",
|
||||
normalizedScanProduced: false,
|
||||
},
|
||||
latencyMs: number(source.latency_ms, "latency_ms"),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -138,9 +138,11 @@ export function DatasetGatewayWorkspace() {
|
|||
masks: {
|
||||
currentGround: comparisonAligned?.currentGround ?? emptyMask,
|
||||
currentAssigned: comparisonAligned?.evaluated ?? emptyMask,
|
||||
candidateGround: comparisonAligned?.groundTruthGround ?? emptyMask,
|
||||
candidateAssigned: comparisonAligned?.evaluated ?? emptyMask,
|
||||
disagreement: comparisonAligned?.disagreement ?? emptyMask,
|
||||
candidateGround: comparisonAligned?.patchworkGround ?? emptyMask,
|
||||
candidateAssigned: comparisonAligned?.patchworkAssigned ?? emptyMask,
|
||||
disagreement: comparisonAligned?.currentDisagreement ?? emptyMask,
|
||||
candidateDisagreement:
|
||||
comparisonAligned?.patchworkDisagreement ?? emptyMask,
|
||||
groundTruthGround: preview.groundTruthGround,
|
||||
},
|
||||
};
|
||||
|
|
@ -301,7 +303,9 @@ export function DatasetGatewayWorkspace() {
|
|||
...(comparison
|
||||
? ([
|
||||
["current", "Current"],
|
||||
["disagreement", "Ошибки"],
|
||||
["candidate", "Patchwork++"],
|
||||
["disagreement", "Ошибки Current"],
|
||||
["candidate-disagreement", "Ошибки PW++"],
|
||||
] as const)
|
||||
: []),
|
||||
["intensity", "Remission"],
|
||||
|
|
@ -320,20 +324,42 @@ export function DatasetGatewayWorkspace() {
|
|||
{comparison ? (
|
||||
<dl className="dataset-preview__metrics">
|
||||
<div>
|
||||
<dt>Ground IoU</dt>
|
||||
<dd>{(comparison.metrics.groundIou * 100).toFixed(1)}%</dd>
|
||||
<dt>Current · IoU</dt>
|
||||
<dd>{(comparison.metrics.current.groundIou * 100).toFixed(1)}%</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Precision</dt>
|
||||
<dd>{(comparison.metrics.precision * 100).toFixed(1)}%</dd>
|
||||
<dt>Current · Recall</dt>
|
||||
<dd>{(comparison.metrics.current.recall * 100).toFixed(1)}%</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Recall</dt>
|
||||
<dd>{(comparison.metrics.recall * 100).toFixed(1)}%</dd>
|
||||
<dt>Current · Natural</dt>
|
||||
<dd>
|
||||
{(comparison.metrics.current.naturalGroundRecall * 100).toFixed(1)}%
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Latency</dt>
|
||||
<dd>{comparison.latencyMs.toFixed(0)} ms</dd>
|
||||
<dt>Current · Latency</dt>
|
||||
<dd>{comparison.latencyMs.current.toFixed(0)} ms</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Patchwork++ · IoU</dt>
|
||||
<dd>
|
||||
{(comparison.metrics.patchworkpp.groundIou * 100).toFixed(1)}%
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Patchwork++ · Recall</dt>
|
||||
<dd>{(comparison.metrics.patchworkpp.recall * 100).toFixed(1)}%</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Patchwork++ · Natural</dt>
|
||||
<dd>
|
||||
{(comparison.metrics.patchworkpp.naturalGroundRecall * 100).toFixed(1)}%
|
||||
</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Patchwork++ · Latency</dt>
|
||||
<dd>{comparison.latencyMs.patchworkpp.toFixed(1)} ms</dd>
|
||||
</div>
|
||||
</dl>
|
||||
) : comparisonError ? (
|
||||
|
|
@ -362,9 +388,19 @@ export function DatasetGatewayWorkspace() {
|
|||
<span><i className="dataset-legend-ground" />current ground</span>
|
||||
<span><i className="dataset-legend-other" />current non-ground</span>
|
||||
</>
|
||||
) : previewMode === "candidate" ? (
|
||||
<>
|
||||
<span><i className="dataset-legend-ground" />Patchwork++ ground</span>
|
||||
<span><i className="dataset-legend-other" />Patchwork++ non-ground</span>
|
||||
</>
|
||||
) : previewMode === "disagreement" ? (
|
||||
<>
|
||||
<span><i className="dataset-legend-error" />ошибка относительно labels</span>
|
||||
<span><i className="dataset-legend-error" />ошибка Current</span>
|
||||
<span><i className="dataset-legend-other" />совпадение с labels</span>
|
||||
</>
|
||||
) : previewMode === "candidate-disagreement" ? (
|
||||
<>
|
||||
<span><i className="dataset-legend-error" />ошибка Patchwork++</span>
|
||||
<span><i className="dataset-legend-other" />совпадение</span>
|
||||
</>
|
||||
) : (
|
||||
|
|
@ -380,6 +416,14 @@ export function DatasetGatewayWorkspace() {
|
|||
<p>{previewError ?? "Проверяем point alignment и разметку."}</p>
|
||||
</div>
|
||||
)}
|
||||
{comparison ? (
|
||||
<p className="dataset-preview__comparison-note">
|
||||
Patchwork++ использует опубликованную высоту VLS-128{" "}
|
||||
{comparison.inputProfile.sensorHeightM.toFixed(2)} м; это
|
||||
algorithm-specific admission, а не полный vehicle TF и не
|
||||
normalized scan.
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ export type LidarGroundViewMode =
|
|||
| "current"
|
||||
| "candidate"
|
||||
| "disagreement"
|
||||
| "candidate-disagreement"
|
||||
| "semantic"
|
||||
| "ground-truth";
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ export interface LidarGroundPointCloudFrame {
|
|||
candidateGround: number[];
|
||||
candidateAssigned: number[];
|
||||
disagreement: number[];
|
||||
candidateDisagreement?: number[];
|
||||
groundTruthGround?: number[];
|
||||
};
|
||||
}
|
||||
|
|
@ -119,6 +121,15 @@ function frameColors(
|
|||
disagreement ? 0.31 : 0.29,
|
||||
disagreement ? 0.22 : 0.35,
|
||||
);
|
||||
} else if (mode === "candidate-disagreement") {
|
||||
const disagreement = frame.masks.candidateDisagreement?.[index] === 1;
|
||||
setRgb(
|
||||
colors,
|
||||
offset,
|
||||
disagreement ? 1 : 0.24,
|
||||
disagreement ? 0.31 : 0.29,
|
||||
disagreement ? 0.22 : 0.35,
|
||||
);
|
||||
} else {
|
||||
setRgb(colors, offset, candidate ? 0.24 : 0.29, candidate ? 0.84 : 0.36, 0.43);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -194,18 +194,22 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
|||
);
|
||||
});
|
||||
|
||||
test("decodes a point-aligned current-vs-ground-truth comparison", async () => {
|
||||
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
|
||||
const payload = {
|
||||
schema_version: "missioncore.dataset-ground-comparison-preview/v1",
|
||||
schema_version: "missioncore.dataset-ground-comparison-preview/v2",
|
||||
source_id: "goose-3d/v2025-08-22",
|
||||
frame_id: "frame-1",
|
||||
sampling: "deterministic-even-index",
|
||||
point_count: 2,
|
||||
current_ground: [1, 1],
|
||||
patchwork_ground: [1, 0],
|
||||
patchwork_assigned: [1, 1],
|
||||
ground_truth_ground: [1, 0],
|
||||
evaluated: [1, 1],
|
||||
disagreement: [0, 1],
|
||||
current_disagreement: [0, 1],
|
||||
patchwork_disagreement: [0, 0],
|
||||
metrics: {
|
||||
current: {
|
||||
true_positive: 1,
|
||||
false_positive: 1,
|
||||
false_negative: 0,
|
||||
|
|
@ -219,27 +223,92 @@ test("decodes a point-aligned current-vs-ground-truth comparison", async () => {
|
|||
natural_ground_recall: 0,
|
||||
obstacle_non_ground_recall: 1,
|
||||
},
|
||||
latency_ms: 12.5,
|
||||
provider: {
|
||||
patchworkpp: {
|
||||
true_positive: 1,
|
||||
false_positive: 0,
|
||||
false_negative: 0,
|
||||
true_negative: 1,
|
||||
precision: 1,
|
||||
recall: 1,
|
||||
f1: 1,
|
||||
ground_iou: 1,
|
||||
accuracy: 1,
|
||||
artificial_ground_recall: 1,
|
||||
natural_ground_recall: 1,
|
||||
obstacle_non_ground_recall: 1,
|
||||
},
|
||||
},
|
||||
latency_ms: {
|
||||
current: 12.5,
|
||||
patchworkpp: 0.4,
|
||||
},
|
||||
patchwork_assigned_fraction: 1,
|
||||
providers: {
|
||||
current: {
|
||||
provider_id: "missioncore-local-percentile-ground/v1",
|
||||
implementation_sha256: "a".repeat(64),
|
||||
ground_truth: false,
|
||||
},
|
||||
patchworkpp: {
|
||||
provider_id: "patchworkpp/v1.4.1",
|
||||
source_url: "https://github.com/url-kaist/patchwork-plusplus",
|
||||
source_tag: "v1.4.1",
|
||||
source_commit: "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||
binding_version: "0.0.1",
|
||||
binary_sha256: "b".repeat(64),
|
||||
platform: "linux",
|
||||
machine: "x86_64",
|
||||
ground_truth: false,
|
||||
},
|
||||
},
|
||||
input_profile: {
|
||||
schema_version: "missioncore.goose-patchwork-profile/v1",
|
||||
source_id: "goose-3d/v2025-08-22",
|
||||
representation: "native-scan",
|
||||
sensor_frame: {
|
||||
frame_id: "sensor/lidar/vls128_roof",
|
||||
handedness: "right",
|
||||
x: "forward",
|
||||
y: "left",
|
||||
z: "up",
|
||||
one_revolution: true,
|
||||
},
|
||||
height: {
|
||||
base_link_above_ground_m: 0.64,
|
||||
lidar_above_base_link_m: 1.6,
|
||||
sensor_above_ground_m: 2.24,
|
||||
evidence: "published-dimensioned-schematic",
|
||||
},
|
||||
scope: {
|
||||
patchworkpp_eligible: true,
|
||||
normalized_scan_produced: false,
|
||||
complete_vehicle_transform_known: false,
|
||||
deskew_claimed: false,
|
||||
},
|
||||
},
|
||||
safety: {
|
||||
qualification_only: true,
|
||||
navigation_or_safety_accepted: false,
|
||||
},
|
||||
};
|
||||
const parsed = parseDatasetGroundComparison(payload);
|
||||
assert.equal(parsed.metrics.groundIou, 0.5);
|
||||
assert.deepEqual(parsed.disagreement, [0, 1]);
|
||||
assert.equal(parsed.metrics.current.groundIou, 0.5);
|
||||
assert.equal(parsed.metrics.patchworkpp.groundIou, 1);
|
||||
assert.deepEqual(parsed.patchworkDisagreement, [0, 0]);
|
||||
assert.equal(parsed.inputProfile.sensorHeightM, 2.24);
|
||||
|
||||
const fetched = await fetchDatasetGroundComparison({
|
||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
||||
});
|
||||
assert.equal(fetched.latencyMs, 12.5);
|
||||
assert.equal(fetched.latencyMs.patchworkpp, 0.4);
|
||||
|
||||
payload.disagreement.pop();
|
||||
payload.patchwork_disagreement.pop();
|
||||
assert.throws(
|
||||
() => parseDatasetGroundComparison(payload),
|
||||
DatasetGatewayContractError,
|
||||
);
|
||||
payload.patchwork_disagreement.push(0);
|
||||
payload.input_profile.height.sensor_above_ground_m = 2.25;
|
||||
assert.throws(
|
||||
() => parseDatasetGroundComparison(payload),
|
||||
DatasetGatewayContractError,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Date: 2026-07-25
|
||||
Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete;
|
||||
Dataset Gateway S0 implemented, first real GOOSE import pending worker D storage
|
||||
Dataset Gateway first-frame Current/Patchwork++ A/B complete
|
||||
Scope: real scanner records, replay and future live shadow processing
|
||||
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
||||
|
||||
|
|
@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
|
|||
domain adaptation after a public baseline proves that the pipeline and metric
|
||||
harness work.
|
||||
|
||||
### L2.5 — Dataset Gateway — S0 complete, real import pending
|
||||
### L2.5 — Dataset Gateway — first-frame A/B complete
|
||||
|
||||
- [x] Define separate `native-scan`, `normalized-scan` and
|
||||
`rolling-local-map` representations.
|
||||
|
|
@ -304,13 +304,16 @@ harness work.
|
|||
per-point time and line/ring fields.
|
||||
- [x] Require operator-admitted storage under
|
||||
`D:\NDC_MISSIONCORE\datasets`.
|
||||
- [ ] Configure the worker dataset root and record disk/resource baseline.
|
||||
- [ ] Download only the 3.3 GB GOOSE validation archive first and record its
|
||||
- [x] Configure the worker dataset root and record disk/resource baseline.
|
||||
- [x] Download only the 3.3 GB GOOSE validation archive first and record its
|
||||
hash/license/provenance.
|
||||
- [ ] Show one real labeled revolution in React with native remission and
|
||||
- [x] Show one real labeled revolution in React with native remission and
|
||||
ground-truth superclass coloring.
|
||||
- [ ] Admit an explicit frame/mounting profile before normalization.
|
||||
- [ ] Run current ground and Patchwork++ against GOOSE ground truth.
|
||||
- [x] Admit the published Patchwork-specific axes/height profile without
|
||||
claiming full vehicle TF, deskew or a general normalized scan.
|
||||
- [x] Run current ground and Patchwork++ against the same GOOSE native scan and
|
||||
ground truth; keep the result one-frame diagnostic.
|
||||
- [ ] Expand Current/Patchwork++ qualification to the validation split.
|
||||
- [ ] Add named range/FOV/density/noise/dropout degradation profiles without
|
||||
overwriting the native frame.
|
||||
|
||||
|
|
@ -387,8 +390,8 @@ The near-term value is not a prettier point cloud:
|
|||
- hardware selection becomes evidence-driven: a future vehicle LiDAR is
|
||||
accepted by its timing/fields/profile, not by vendor marketing.
|
||||
|
||||
The highest-value immediate work is the worker-side GOOSE validation import,
|
||||
the first labeled native scan in React and an accuracy-bearing ground A/B.
|
||||
LiDAR-native detection follows on the same gateway. Nvblox and alternative
|
||||
SLAM remain later because their timing, pose and scan-geometry gates are not yet
|
||||
satisfied.
|
||||
The highest-value immediate work is now the validation-split
|
||||
Current/Patchwork++ gate plus named range/FOV/density/noise/dropout degradation
|
||||
profiles. LiDAR-native detection follows on the same gateway. Nvblox and
|
||||
alternative SLAM remain later because their timing, pose and scan-geometry
|
||||
gates are not yet satisfied.
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ Implemented now:
|
|||
`Открыть` action;
|
||||
- one admitted GOOSE validation frame with source colors, normalized remission
|
||||
and an independent ground-truth view;
|
||||
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
||||
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
||||
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
||||
- a reproducible Current/Patchwork++ A/B artifact with point-aligned masks,
|
||||
disagreement views, independent-label metrics, latency and exact provider
|
||||
identities;
|
||||
- a separate **Парк → Диагностика LiDAR** surface containing only real sensor
|
||||
recordings and their operational evidence.
|
||||
|
||||
|
|
@ -152,11 +158,13 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
|||
4. [Done] Import one labeled frame and expose it in React as `native-scan`.
|
||||
5. [Done] Show native remission, the original semantic palette and
|
||||
ground-truth superclass coloring.
|
||||
6. Add a declared GOOSE frame/mounting profile and produce `normalized-scan`.
|
||||
7. [Current baseline done; Patchwork++ blocked on mounting evidence] Run the
|
||||
current ground heuristic and Patchwork++ against independent labels.
|
||||
6. [Patchwork-specific profile done; general normalization still pending] Admit
|
||||
the published VLS-128 axes and physical height without claiming a complete
|
||||
numeric vehicle TF or deskewed `normalized-scan`.
|
||||
7. [One-frame A/B done] Run the current ground heuristic and pinned
|
||||
Patchwork++ against the same native scan and independent labels.
|
||||
8. Add sensor-degradation profiles for range, FOV, density, noise and dropout.
|
||||
9. Only after the one-frame contract passes, expand to the validation split and
|
||||
9. Expand the A/B to the validation split; only after that qualification gate,
|
||||
add a rolling-map sequence with localization evidence.
|
||||
|
||||
## Acceptance checklist
|
||||
|
|
@ -172,9 +180,11 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
|||
- [x] Worker D root configured.
|
||||
- [x] GOOSE validation archive hash recorded.
|
||||
- [x] First real labeled frame visible in React.
|
||||
- [ ] Coordinate and mounting profile admitted.
|
||||
- [x] Patchwork-specific axes and physical-height profile admitted.
|
||||
- [ ] Complete vehicle transform and general `normalized-scan` admitted.
|
||||
- [x] Current local-percentile baseline measured against ground truth.
|
||||
- [ ] Patchwork++ accuracy measured against ground truth.
|
||||
- [x] Patchwork++ one-frame accuracy measured against ground truth.
|
||||
- [ ] Current/Patchwork++ validation-split gate qualified.
|
||||
- [ ] Sensor-degradation matrix qualified.
|
||||
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
||||
|
||||
|
|
@ -229,10 +239,54 @@ previous exact-radius result. This removes the previous all-points scan for
|
|||
every occupied cell, but the measured latency still classifies it as a
|
||||
diagnostic baseline rather than an onboard candidate.
|
||||
|
||||
Patchwork++ is intentionally not scored yet. The validation ZIP contains XYZI,
|
||||
labels, mapping, LICENSE and CHANGELOG but no numeric TF/mounting calibration.
|
||||
GOOSE documents the VLS-128 as a roof LiDAR and publishes a separate MuCAR-3 TF
|
||||
tree, but the graph image alone is not physical-height evidence. The next gate
|
||||
is to admit the numeric transform from `base_link_ground` to
|
||||
`sensor/lidar/vls128_roof`, declare the source axis convention, and only then
|
||||
run Patchwork++.
|
||||
## First Patchwork++ A/B result
|
||||
|
||||
GOOSE's dimensioned MuCAR-3 schematic provides the two vertical dimensions
|
||||
needed by this algorithm-specific gate: `base_link` is `0.64 m` above ground
|
||||
and the VLS-128 optical center is `1.60 m` above `base_link`. The admitted
|
||||
Patchwork++ height is therefore `2.24 m`. The same published schematic declares
|
||||
the sensor axes as `x` forward, `y` left and `z` up. An independent fit to
|
||||
GOOSE-labeled near-field ground observed a `-2.18 .. -2.14 m` intercept; this
|
||||
was a non-calibrating cross-check, not the source of the height.
|
||||
|
||||
This narrowly admits the input required by Patchwork++ on the native
|
||||
sensor-centric scan. It does **not** claim a complete numeric vehicle transform,
|
||||
per-point timing, deskew or a general `normalized-scan`.
|
||||
|
||||
Official Patchwork++ `v1.4.1`, source commit
|
||||
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`, was run against the same first
|
||||
frame and independent labels as Current:
|
||||
|
||||
| Metric | Current | Patchwork++ |
|
||||
| --- | ---: | ---: |
|
||||
| Ground IoU | `49.6165%` | `60.3702%` |
|
||||
| Precision | `73.5892%` | `72.6612%` |
|
||||
| Recall | `60.3659%` | `78.1130%` |
|
||||
| F1 | `66.3249%` | `75.2886%` |
|
||||
| Artificial-ground recall | `95.2364%` | `98.9688%` |
|
||||
| Natural-ground recall | `49.4516%` | `71.5853%` |
|
||||
| Obstacle non-ground recall | `79.6145%` | `92.1729%` |
|
||||
| Worker latency | `3966.53 ms` | `15.89 ms` |
|
||||
|
||||
Reproducibility pins:
|
||||
|
||||
- benchmark identity:
|
||||
`2a6d05f54a9e2ac727d9c850c1133f2fb539bfeddbd97c07cfd198ccb239162c`;
|
||||
- full point-aligned prediction SHA-256:
|
||||
`190f455c6e47911921b7f6454913e1bb2d0b8b5814ee2e34d7b63295afddc7ef`;
|
||||
- bounded browser preview SHA-256:
|
||||
`e23ca573103faf931523415e6c263248737eb6600f482b63de685b725bbc28c3`;
|
||||
- Patchwork++ binary SHA-256:
|
||||
`be8038b2098c83fe53841aa8ae19e362910e9056fe0ee7b9304c1ee5c5941094`.
|
||||
|
||||
Patchwork++ wins this frame by `10.75` percentage points of Ground IoU, raises
|
||||
natural-ground recall by `22.13` points and is roughly `250x` faster in this
|
||||
run. The result is deliberately `one-frame-diagnostic`: it makes Patchwork++
|
||||
the candidate for validation-split and degradation qualification, not an
|
||||
accepted navigation or safety provider.
|
||||
|
||||
Primary source evidence:
|
||||
|
||||
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/);
|
||||
- [GOOSE paper, Figure 3](https://arxiv.org/pdf/2310.16788);
|
||||
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1).
|
||||
|
|
|
|||
|
|
@ -87,24 +87,34 @@ unavailable rather than inventing timestamps.
|
|||
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
||||
robustness experiments, but it cannot recreate lost timestamps, occlusions
|
||||
or material response.
|
||||
- Patchwork++ becomes eligible for a real quality gate only on a sensor-centric
|
||||
scan with declared scan geometry and physical mounting height plus
|
||||
independent labels.
|
||||
- Patchwork++ becomes eligible for an algorithm-specific quality gate on a
|
||||
sensor-centric scan with declared axes, physical mounting height and
|
||||
independent labels. That narrow admission does not itself create a general
|
||||
`normalized-scan`.
|
||||
- Stable operator visualization is owned by rolling-map policy, not by the
|
||||
ground classifier.
|
||||
- Public labels may qualify an algorithm independently of the production
|
||||
sensor. The first GOOSE frame exposed a `49.62%` Ground IoU and only `49.45%`
|
||||
natural-ground recall for the current local-percentile baseline; these are
|
||||
diagnostic results, not production promotion.
|
||||
- A dataset label contract does not imply a mounting contract. Patchwork++
|
||||
remains blocked until the numeric GOOSE roof-LiDAR transform and physical
|
||||
height are admitted from source evidence.
|
||||
- A dataset label contract does not imply a mounting contract. For MuCAR-3,
|
||||
GOOSE's published dimensioned schematic admits the Patchwork-specific
|
||||
`2.24 m` sensor height (`0.64 + 1.60 m`) and `x-forward / y-left / z-up`
|
||||
axes. The complete numeric vehicle transform, deskew and general
|
||||
`normalized-scan` remain unavailable.
|
||||
- On the first independently labeled frame, pinned Patchwork++ `v1.4.1`
|
||||
achieved `60.37%` Ground IoU, `71.59%` natural-ground recall and `15.89 ms`
|
||||
worker latency versus Current's `49.62%`, `49.45%` and `3966.53 ms`.
|
||||
This is a one-frame diagnostic candidate decision, not production promotion.
|
||||
|
||||
## Primary references
|
||||
|
||||
- [GOOSE dataset structure](https://goose-dataset.de/docs/dataset-structure/)
|
||||
- [GOOSE setup and archive sizes](https://goose-dataset.de/docs/setup/)
|
||||
- [GOOSE 3D challenge ontology](https://goose-dataset.de/docs/3d-semantic-segmentation-challenge/)
|
||||
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
|
||||
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
|
||||
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
|
||||
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
||||
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
||||
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
||||
|
|
|
|||
|
|
@ -1,18 +1,14 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any, Final
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -25,9 +21,21 @@ from k1link.ground_segmentation import (
|
|||
GroundSegmenter,
|
||||
LocalPercentileGroundSegmenter,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
PATCHWORKPP_SOURCE_TAG as PATCHWORKPP_SOURCE_TAG,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
GroundSegmentationError as LidarGroundError,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter,
|
||||
)
|
||||
|
||||
from .lidar_contract import LidarContractError, sensor_frame_xyzi
|
||||
from .lidar_replay import LidarReplayPackV2
|
||||
|
|
@ -42,107 +50,9 @@ LIDAR_GROUND_MANIFEST_NAME: Final = "manifest.json"
|
|||
LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz"
|
||||
MAX_GROUND_FRAME_POINTS: Final = 200_000
|
||||
|
||||
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
|
||||
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
|
||||
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||
|
||||
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
|
||||
_ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$")
|
||||
|
||||
|
||||
class PatchworkPPGroundSegmenter:
|
||||
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
module: ModuleType,
|
||||
profile: GroundBenchmarkProfile,
|
||||
*,
|
||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||
) -> None:
|
||||
if _GIT_SHA1.fullmatch(source_commit) is None:
|
||||
raise LidarGroundError("Patchwork++ source commit is invalid")
|
||||
if source_tag != PATCHWORKPP_SOURCE_TAG:
|
||||
raise LidarGroundError("Patchwork++ source tag is not admitted")
|
||||
module_path_value = getattr(module, "__file__", None)
|
||||
if not isinstance(module_path_value, str):
|
||||
raise LidarGroundError("Patchwork++ module has no verifiable binary")
|
||||
module_path = Path(module_path_value).resolve(strict=True)
|
||||
params = module.Parameters()
|
||||
params.sensor_height = profile.patchwork_sensor_height_proxy_m
|
||||
params.min_range = profile.patchwork_minimum_range_m
|
||||
params.max_range = profile.patchwork_maximum_range_m
|
||||
params.enable_RNR = True
|
||||
params.enable_RVPF = True
|
||||
params.enable_TGR = True
|
||||
params.verbose = False
|
||||
self._estimator = module.patchworkpp(params)
|
||||
self._identity = {
|
||||
"provider_id": "patchworkpp/v1.4.1",
|
||||
"source_url": PATCHWORKPP_SOURCE_URL,
|
||||
"source_tag": source_tag,
|
||||
"source_commit": source_commit,
|
||||
"binding_version": str(getattr(module, "__version__", "unknown")),
|
||||
"binary_sha256": _sha256(module_path),
|
||||
"platform": platform.system().lower(),
|
||||
"machine": platform.machine().lower(),
|
||||
"ground_truth": False,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls,
|
||||
profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
*,
|
||||
module_name: str = "pypatchworkpp",
|
||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||
) -> PatchworkPPGroundSegmenter:
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
except ImportError as exc:
|
||||
raise LidarGroundError("Pinned Patchwork++ Python binding is unavailable") from exc
|
||||
return cls(
|
||||
module,
|
||||
profile,
|
||||
source_commit=source_commit,
|
||||
source_tag=source_tag,
|
||||
)
|
||||
|
||||
@property
|
||||
def identity(self) -> Mapping[str, object]:
|
||||
return self._identity
|
||||
|
||||
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
|
||||
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
|
||||
started = time.perf_counter_ns()
|
||||
self._estimator.estimateGround(points)
|
||||
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
|
||||
ground_indices = np.asarray(
|
||||
self._estimator.getGroundIndices(),
|
||||
dtype=np.int64,
|
||||
).reshape((-1,))
|
||||
nonground_indices = np.asarray(
|
||||
self._estimator.getNongroundIndices(),
|
||||
dtype=np.int64,
|
||||
).reshape((-1,))
|
||||
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
|
||||
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
|
||||
if np.intersect1d(ground_indices, nonground_indices).size:
|
||||
raise LidarGroundError("Patchwork++ assigned one point twice")
|
||||
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
||||
assigned = np.zeros(points.shape[0], dtype=np.bool_)
|
||||
ground[ground_indices] = True
|
||||
assigned[ground_indices] = True
|
||||
assigned[nonground_indices] = True
|
||||
return GroundSegmentation(
|
||||
ground_mask=ground,
|
||||
assigned_mask=assigned,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
|
||||
|
||||
class LidarGroundBenchmarkV1:
|
||||
|
|
|
|||
|
|
@ -15,9 +15,17 @@ from k1link.datasets.gateway import (
|
|||
read_semantic_kitti_frame,
|
||||
)
|
||||
from k1link.datasets.goose_benchmark import (
|
||||
GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
|
||||
GOOSE_GROUND_AB_PREVIEW_SCHEMA,
|
||||
GOOSE_GROUND_BENCHMARK_SCHEMA,
|
||||
GOOSE_GROUND_PREVIEW_SCHEMA,
|
||||
benchmark_goose_current_ground,
|
||||
benchmark_goose_patchwork_ground,
|
||||
)
|
||||
from k1link.datasets.goose_profile import (
|
||||
DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||
GOOSE_PATCHWORK_PROFILE_SCHEMA,
|
||||
GoosePatchworkProfile,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -25,9 +33,15 @@ __all__ = [
|
|||
"DatasetAdmissionError",
|
||||
"DatasetFrameError",
|
||||
"DatasetPointFrame",
|
||||
"DEFAULT_GOOSE_PATCHWORK_PROFILE",
|
||||
"GOOSE_GROUND_AB_BENCHMARK_SCHEMA",
|
||||
"GOOSE_GROUND_AB_PREVIEW_SCHEMA",
|
||||
"GOOSE_GROUND_BENCHMARK_SCHEMA",
|
||||
"GOOSE_GROUND_PREVIEW_SCHEMA",
|
||||
"GOOSE_PATCHWORK_PROFILE_SCHEMA",
|
||||
"GoosePatchworkProfile",
|
||||
"benchmark_goose_current_ground",
|
||||
"benchmark_goose_patchwork_ground",
|
||||
"configured_dataset_admission_manifest",
|
||||
"configured_dataset_ground_preview",
|
||||
"configured_dataset_preview",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ from typing import Annotated
|
|||
import typer
|
||||
|
||||
from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation
|
||||
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
|
||||
from k1link.datasets.goose_benchmark import (
|
||||
benchmark_goose_current_ground,
|
||||
benchmark_goose_patchwork_ground,
|
||||
)
|
||||
|
||||
app = typer.Typer(
|
||||
add_completion=False,
|
||||
|
|
@ -63,5 +66,22 @@ def benchmark_goose_current_ground_command(
|
|||
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
@app.command("benchmark-goose-patchwork-ground")
|
||||
def benchmark_goose_patchwork_ground_command(
|
||||
dataset_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||
],
|
||||
) -> None:
|
||||
"""Compare current and pinned Patchwork++ providers against GOOSE labels."""
|
||||
|
||||
try:
|
||||
report = benchmark_goose_patchwork_ground(dataset_root)
|
||||
except (GooseAdmissionError, ValueError) as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
|
|
|||
|
|
@ -16,10 +16,12 @@ from typing import Any, Final, Literal
|
|||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
|
||||
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
||||
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
|
||||
DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST"
|
||||
DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW"
|
||||
|
|
@ -316,8 +318,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
|||
raise DatasetAdmissionError("dataset admission status is incompatible")
|
||||
storage = _object(document["storage"], "storage")
|
||||
if (
|
||||
set(storage)
|
||||
!= {"policy", "admitted", "canonical_root", "path_exposed"}
|
||||
set(storage) != {"policy", "admitted", "canonical_root", "path_exposed"}
|
||||
or storage.get("policy") != "worker-d-only"
|
||||
or storage.get("admitted") is not True
|
||||
or storage.get("canonical_root") is not True
|
||||
|
|
@ -338,8 +339,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
|||
"vendor_checksum_available",
|
||||
}
|
||||
or archive.get("filename") != "goose_3d_val.zip"
|
||||
or archive.get("source_url")
|
||||
!= "https://goose-dataset.de/storage/goose_3d_val.zip"
|
||||
or archive.get("source_url") != "https://goose-dataset.de/storage/goose_3d_val.zip"
|
||||
or archive.get("vendor_checksum_available") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset archive admission is incompatible")
|
||||
|
|
@ -385,9 +385,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
|||
):
|
||||
raise DatasetAdmissionError("dataset preview identity is incompatible")
|
||||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||
source_point_count = _positive_integer(
|
||||
document.get("source_point_count"), "source_point_count"
|
||||
)
|
||||
source_point_count = _positive_integer(document.get("source_point_count"), "source_point_count")
|
||||
if point_count > 50_000 or point_count > source_point_count:
|
||||
raise DatasetAdmissionError("dataset preview point count is incompatible")
|
||||
points = document.get("points_xyz_m")
|
||||
|
|
@ -456,20 +454,28 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
|||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||
if point_count > 50_000:
|
||||
raise DatasetAdmissionError("dataset ground preview point count is incompatible")
|
||||
for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"):
|
||||
for key in (
|
||||
"current_ground",
|
||||
"patchwork_ground",
|
||||
"patchwork_assigned",
|
||||
"ground_truth_ground",
|
||||
"evaluated",
|
||||
"current_disagreement",
|
||||
"patchwork_disagreement",
|
||||
):
|
||||
values = document.get(key)
|
||||
if (
|
||||
not isinstance(values, list)
|
||||
or len(values) != point_count
|
||||
or any(
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or value not in (0, 1)
|
||||
not isinstance(value, int) or isinstance(value, bool) or value not in (0, 1)
|
||||
for value in values
|
||||
)
|
||||
):
|
||||
raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible")
|
||||
metrics = _object(document.get("metrics"), "metrics")
|
||||
metrics_by_provider = _object(document.get("metrics"), "metrics")
|
||||
if set(metrics_by_provider) != {"current", "patchworkpp"}:
|
||||
raise DatasetAdmissionError("dataset ground provider metrics are incompatible")
|
||||
expected_metrics = {
|
||||
"true_positive",
|
||||
"false_positive",
|
||||
|
|
@ -484,40 +490,88 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
|||
"natural_ground_recall",
|
||||
"obstacle_non_ground_recall",
|
||||
}
|
||||
for provider_id in ("current", "patchworkpp"):
|
||||
metrics = _object(metrics_by_provider[provider_id], f"metrics.{provider_id}")
|
||||
if set(metrics) != expected_metrics:
|
||||
raise DatasetAdmissionError("dataset ground metrics are incompatible")
|
||||
for key, value in metrics.items():
|
||||
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
|
||||
_nonnegative_integer(value, f"metrics.{key}")
|
||||
_nonnegative_integer(value, f"metrics.{provider_id}.{key}")
|
||||
elif (
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or not np.isfinite(value)
|
||||
or not 0 <= value <= 1
|
||||
):
|
||||
raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible")
|
||||
latency = document.get("latency_ms")
|
||||
raise DatasetAdmissionError(
|
||||
f"dataset ground metric {provider_id}.{key} is incompatible"
|
||||
)
|
||||
latency = _object(document.get("latency_ms"), "latency_ms")
|
||||
if set(latency) != {"current", "patchworkpp"}:
|
||||
raise DatasetAdmissionError("dataset ground latency providers are incompatible")
|
||||
for provider_id, value in latency.items():
|
||||
if (
|
||||
not isinstance(latency, (int, float))
|
||||
or isinstance(latency, bool)
|
||||
or not np.isfinite(latency)
|
||||
or latency < 0
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or not np.isfinite(value)
|
||||
or value < 0
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground latency is incompatible")
|
||||
provider = _object(document.get("provider"), "provider")
|
||||
raise DatasetAdmissionError(f"dataset ground latency {provider_id} is incompatible")
|
||||
providers = _object(document.get("providers"), "providers")
|
||||
if set(providers) != {"current", "patchworkpp"}:
|
||||
raise DatasetAdmissionError("dataset ground providers are incompatible")
|
||||
current_provider = _object(providers["current"], "providers.current")
|
||||
if (
|
||||
set(provider) != {"provider_id", "implementation_sha256", "ground_truth"}
|
||||
or provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
|
||||
or provider.get("ground_truth") is not False
|
||||
set(current_provider) != {"provider_id", "implementation_sha256", "ground_truth"}
|
||||
or current_provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
|
||||
or current_provider.get("ground_truth") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground provider is incompatible")
|
||||
digest = provider.get("implementation_sha256")
|
||||
digest = current_provider.get("implementation_sha256")
|
||||
if (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground provider digest is incompatible")
|
||||
patchwork_provider = _object(providers["patchworkpp"], "providers.patchworkpp")
|
||||
if (
|
||||
set(patchwork_provider)
|
||||
!= {
|
||||
"provider_id",
|
||||
"source_url",
|
||||
"source_tag",
|
||||
"source_commit",
|
||||
"binding_version",
|
||||
"binary_sha256",
|
||||
"platform",
|
||||
"machine",
|
||||
"ground_truth",
|
||||
}
|
||||
or patchwork_provider.get("provider_id") != "patchworkpp/v1.4.1"
|
||||
or patchwork_provider.get("source_tag") != "v1.4.1"
|
||||
or patchwork_provider.get("source_commit") != "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||
or patchwork_provider.get("ground_truth") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset Patchwork++ provider is incompatible")
|
||||
binary_digest = patchwork_provider.get("binary_sha256")
|
||||
if (
|
||||
not isinstance(binary_digest, str)
|
||||
or len(binary_digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in binary_digest)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset Patchwork++ binary digest is incompatible")
|
||||
assigned_fraction = document.get("patchwork_assigned_fraction")
|
||||
if (
|
||||
not isinstance(assigned_fraction, (int, float))
|
||||
or isinstance(assigned_fraction, bool)
|
||||
or not np.isfinite(assigned_fraction)
|
||||
or not 0 <= assigned_fraction <= 1
|
||||
):
|
||||
raise DatasetAdmissionError("dataset Patchwork++ assigned fraction is incompatible")
|
||||
input_profile = _object(document.get("input_profile"), "input_profile")
|
||||
if input_profile != DEFAULT_GOOSE_PATCHWORK_PROFILE.to_dict():
|
||||
raise DatasetAdmissionError("dataset Patchwork++ input profile is incompatible")
|
||||
safety = _object(document.get("safety"), "safety")
|
||||
if safety != {
|
||||
"qualification_only": True,
|
||||
|
|
|
|||
|
|
@ -18,14 +18,22 @@ from k1link.datasets.goose_admission import (
|
|||
GooseAdmissionError,
|
||||
read_goose_label_mapping,
|
||||
)
|
||||
from k1link.datasets.goose_profile import (
|
||||
DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||
GoosePatchworkProfile,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
GroundBenchmarkProfile,
|
||||
GroundSegmenter,
|
||||
LocalPercentileGroundSegmenter,
|
||||
PatchworkPPGroundSegmenter,
|
||||
)
|
||||
|
||||
GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1"
|
||||
GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
||||
GOOSE_GROUND_AB_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-ab-benchmark/v1"
|
||||
GOOSE_GROUND_AB_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||
MAX_PREVIEW_POINTS: Final = 50_000
|
||||
|
||||
|
||||
|
|
@ -135,8 +143,7 @@ def benchmark_goose_current_ground(
|
|||
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
||||
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
||||
"disagreement": (
|
||||
evaluated[indices]
|
||||
& (prediction.ground_mask[indices] != ground_truth[indices])
|
||||
evaluated[indices] & (prediction.ground_mask[indices] != ground_truth[indices])
|
||||
)
|
||||
.astype(np.uint8)
|
||||
.tolist(),
|
||||
|
|
@ -179,6 +186,204 @@ def benchmark_goose_current_ground(
|
|||
return report
|
||||
|
||||
|
||||
def benchmark_goose_patchwork_ground(
|
||||
dataset_root: Path,
|
||||
*,
|
||||
current_profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
patchwork_profile: GoosePatchworkProfile = DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||
patchwork: GroundSegmenter | None = None,
|
||||
patchwork_module_name: str = "pypatchworkpp",
|
||||
preview_points: int = MAX_PREVIEW_POINTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Score current and pinned Patchwork++ providers against one GOOSE frame."""
|
||||
|
||||
root = dataset_root.expanduser().absolute()
|
||||
if not _is_worker_dataset_root(root):
|
||||
raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root")
|
||||
manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json")
|
||||
if manifest["status"] != "frame-ready":
|
||||
raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking")
|
||||
archive_sha256 = str(manifest["archive"]["sha256"])
|
||||
frame_id = str(manifest["frame"]["frame_id"])
|
||||
install = root / "goose-3d/v2025-08-22/installs" / archive_sha256
|
||||
frame_root = install / "frames" / frame_id
|
||||
frame = read_semantic_kitti_frame(
|
||||
frame_root / "points.bin",
|
||||
frame_root / "labels.label",
|
||||
)
|
||||
labels = read_goose_label_mapping(install / "goose_label_mapping.csv")
|
||||
categories = np.asarray(
|
||||
[labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels],
|
||||
dtype=np.uint8,
|
||||
)
|
||||
evaluated = categories != 0
|
||||
ground_truth = (categories == 2) | (categories == 3)
|
||||
xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype(
|
||||
np.float32,
|
||||
copy=False,
|
||||
)
|
||||
current = LocalPercentileGroundSegmenter(current_profile)
|
||||
candidate = (
|
||||
patchwork
|
||||
if patchwork is not None
|
||||
else PatchworkPPGroundSegmenter.load(
|
||||
patchwork_profile,
|
||||
module_name=patchwork_module_name,
|
||||
)
|
||||
)
|
||||
current_result = current.segment(xyzi)
|
||||
candidate_result = candidate.segment(xyzi)
|
||||
for result, label in (
|
||||
(current_result, "current"),
|
||||
(candidate_result, "Patchwork++"),
|
||||
):
|
||||
if result.ground_mask.shape != (frame.point_count,) or result.assigned_mask.shape != (
|
||||
frame.point_count,
|
||||
):
|
||||
raise GooseAdmissionError(f"{label} result is not point-aligned")
|
||||
current_metrics = _ground_metrics(
|
||||
current_result.ground_mask,
|
||||
ground_truth,
|
||||
evaluated,
|
||||
categories,
|
||||
)
|
||||
candidate_metrics = _ground_metrics(
|
||||
candidate_result.ground_mask,
|
||||
ground_truth,
|
||||
evaluated,
|
||||
categories,
|
||||
)
|
||||
candidate_assigned_fraction = float(np.mean(candidate_result.assigned_mask))
|
||||
profile_document = patchwork_profile.to_dict()
|
||||
current_profile_document = _goose_current_profile_document(current_profile)
|
||||
identity_document = {
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"archive_sha256": archive_sha256,
|
||||
"frame_id": frame_id,
|
||||
"source_point_count": frame.point_count,
|
||||
"input_profile": profile_document,
|
||||
"current_profile": current_profile_document,
|
||||
"providers": {
|
||||
"current": dict(current.identity),
|
||||
"patchworkpp": dict(candidate.identity),
|
||||
},
|
||||
}
|
||||
identity_sha256 = _canonical_sha256(identity_document)
|
||||
output_root = install / "benchmarks" / f"ground-ab-{identity_sha256}"
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
report_path = output_root / "report.json"
|
||||
if report_path.is_file():
|
||||
try:
|
||||
existing = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise GooseAdmissionError("existing GOOSE A/B report is invalid") from exc
|
||||
if (
|
||||
not isinstance(existing, dict)
|
||||
or existing.get("schema_version") != GOOSE_GROUND_AB_BENCHMARK_SCHEMA
|
||||
or existing.get("identity_sha256") != identity_sha256
|
||||
or not (output_root / "prediction.npz").is_file()
|
||||
or not (output_root / "preview.json").is_file()
|
||||
):
|
||||
raise GooseAdmissionError("existing GOOSE A/B benchmark is incomplete")
|
||||
return existing
|
||||
|
||||
prediction_path = output_root / "prediction.npz"
|
||||
_atomic_npz(
|
||||
prediction_path,
|
||||
current_ground=current_result.ground_mask.astype(np.uint8),
|
||||
patchwork_ground=candidate_result.ground_mask.astype(np.uint8),
|
||||
patchwork_assigned=candidate_result.assigned_mask.astype(np.uint8),
|
||||
ground_truth_ground=ground_truth.astype(np.uint8),
|
||||
evaluated=evaluated.astype(np.uint8),
|
||||
)
|
||||
sample_count = min(frame.point_count, preview_points)
|
||||
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
|
||||
current_disagreement = evaluated & (current_result.ground_mask != ground_truth)
|
||||
candidate_disagreement = evaluated & (candidate_result.ground_mask != ground_truth)
|
||||
preview = {
|
||||
"schema_version": GOOSE_GROUND_AB_PREVIEW_SCHEMA,
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"frame_id": frame_id,
|
||||
"sampling": "deterministic-even-index",
|
||||
"point_count": sample_count,
|
||||
"current_ground": current_result.ground_mask[indices].astype(np.uint8).tolist(),
|
||||
"patchwork_ground": candidate_result.ground_mask[indices].astype(np.uint8).tolist(),
|
||||
"patchwork_assigned": (candidate_result.assigned_mask[indices].astype(np.uint8).tolist()),
|
||||
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
||||
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
||||
"current_disagreement": (current_disagreement[indices].astype(np.uint8).tolist()),
|
||||
"patchwork_disagreement": (candidate_disagreement[indices].astype(np.uint8).tolist()),
|
||||
"metrics": {
|
||||
"current": current_metrics,
|
||||
"patchworkpp": candidate_metrics,
|
||||
},
|
||||
"latency_ms": {
|
||||
"current": current_result.latency_ms,
|
||||
"patchworkpp": candidate_result.latency_ms,
|
||||
},
|
||||
"patchwork_assigned_fraction": candidate_assigned_fraction,
|
||||
"providers": {
|
||||
"current": dict(current.identity),
|
||||
"patchworkpp": dict(candidate.identity),
|
||||
},
|
||||
"input_profile": profile_document,
|
||||
"safety": {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
preview_path = output_root / "preview.json"
|
||||
_atomic_json(preview_path, preview)
|
||||
winner = (
|
||||
"patchworkpp"
|
||||
if candidate_metrics["ground_iou"] > current_metrics["ground_iou"]
|
||||
else "current"
|
||||
if current_metrics["ground_iou"] > candidate_metrics["ground_iou"]
|
||||
else "tie"
|
||||
)
|
||||
report = {
|
||||
"schema_version": GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity_document,
|
||||
"ground_truth": {
|
||||
"source": "GOOSE point-wise semantic labels",
|
||||
"ground_categories": ["artificial_ground", "natural_ground"],
|
||||
"void_excluded": True,
|
||||
"evaluated_points": int(np.count_nonzero(evaluated)),
|
||||
},
|
||||
"providers": {
|
||||
"current": {
|
||||
"metrics": current_metrics,
|
||||
"latency_ms": current_result.latency_ms,
|
||||
"assigned_fraction": 1.0,
|
||||
},
|
||||
"patchworkpp": {
|
||||
"metrics": candidate_metrics,
|
||||
"latency_ms": candidate_result.latency_ms,
|
||||
"assigned_fraction": candidate_assigned_fraction,
|
||||
},
|
||||
},
|
||||
"artifacts": {
|
||||
"prediction_sha256": _sha256_file(prediction_path),
|
||||
"preview_sha256": _sha256_file(preview_path),
|
||||
},
|
||||
"decision": {
|
||||
"status": "one-frame-diagnostic",
|
||||
"promoted": False,
|
||||
"winner_by_ground_iou": winner,
|
||||
"reason": "one public frame does not qualify a production ground provider",
|
||||
"next_gate": "validation-split qualification with degradation profiles",
|
||||
},
|
||||
"scope": {
|
||||
"patchworkpp_input_admitted": True,
|
||||
"normalized_scan_produced": False,
|
||||
"complete_vehicle_transform_known": False,
|
||||
},
|
||||
}
|
||||
_atomic_json(report_path, report)
|
||||
return report
|
||||
|
||||
|
||||
def _is_worker_dataset_root(root: Path) -> bool:
|
||||
normalized = str(root).replace("\\", "/").rstrip("/").lower()
|
||||
return normalized == "/mnt/d/ndc_missioncore/datasets"
|
||||
|
|
@ -216,6 +421,45 @@ def _binary_metrics(
|
|||
}
|
||||
|
||||
|
||||
def _ground_metrics(
|
||||
prediction: np.ndarray[Any, Any],
|
||||
ground_truth: np.ndarray[Any, Any],
|
||||
evaluated: np.ndarray[Any, Any],
|
||||
categories: np.ndarray[Any, Any],
|
||||
) -> dict[str, float | int]:
|
||||
metrics = _binary_metrics(prediction, ground_truth, evaluated)
|
||||
metrics.update(
|
||||
{
|
||||
"artificial_ground_recall": _recall(prediction, categories == 2),
|
||||
"natural_ground_recall": _recall(prediction, categories == 3),
|
||||
"obstacle_non_ground_recall": _recall(~prediction, categories == 4),
|
||||
}
|
||||
)
|
||||
return metrics
|
||||
|
||||
|
||||
def _goose_current_profile_document(
|
||||
profile: GroundBenchmarkProfile,
|
||||
) -> dict[str, object]:
|
||||
parameters = profile.to_dict()["current_baseline"]
|
||||
return {
|
||||
"schema_version": "missioncore.goose-current-ground-profile/v1",
|
||||
"profile_id": "goose-native-local-percentile/v1",
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"representation": "native-scan",
|
||||
"sensor_frame": "sensor/lidar/vls128_roof",
|
||||
"provider": parameters,
|
||||
"scope": {
|
||||
"complete_vehicle_transform_required": False,
|
||||
"normalized_scan_produced": False,
|
||||
},
|
||||
"authority": {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float:
|
||||
return _ratio(
|
||||
int(np.count_nonzero(prediction & truth)),
|
||||
|
|
@ -259,9 +503,7 @@ def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None:
|
|||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||
if path.exists():
|
||||
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
|
||||
encoded = (
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||
)
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary.write(encoded)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
"""Published, algorithm-scoped GOOSE VLS-128 ground profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
GOOSE_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.goose-patchwork-profile/v1"
|
||||
GOOSE_PAPER_URL: Final = "https://arxiv.org/pdf/2310.16788"
|
||||
GOOSE_PAPER_SHA256: Final = "cb17f38c3ae498917966a63bc0e034cc5dd3fa4ad35873b121592f99026cc77e"
|
||||
GOOSE_TF_DOT_URL: Final = "https://goose-dataset.de/docs/resources/mucar3_tf/mucar3_tf.dot"
|
||||
GOOSE_TF_DOT_SHA256: Final = "1b10e905bcff17b90305ff02734b5e6bb7f5753d9b8e74cd7c43f3d0f32ef469"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GoosePatchworkProfile:
|
||||
"""Narrow admission profile for one native GOOSE scan.
|
||||
|
||||
Figure 3 of the official paper dimensions ``base_link``/INS at 0.64 m
|
||||
above ground and the VLS-128 optical center 1.60 m above ``base_link``.
|
||||
Their sum is the physical height Patchwork++ requires. This does not claim
|
||||
a complete vehicle mounting transform and cannot produce a normalized scan.
|
||||
"""
|
||||
|
||||
profile_id: str = "goose-mucar3-vls128-patchwork/v1"
|
||||
base_link_height_above_ground_m: float = 0.64
|
||||
lidar_height_above_base_link_m: float = 1.60
|
||||
patchwork_sensor_height_proxy_m: float = 2.24
|
||||
patchwork_minimum_range_m: float = 2.7
|
||||
patchwork_maximum_range_m: float = 80.0
|
||||
patchwork_height_evidence: str = "published-dimensioned-schematic"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
values = (
|
||||
self.base_link_height_above_ground_m,
|
||||
self.lidar_height_above_base_link_m,
|
||||
self.patchwork_sensor_height_proxy_m,
|
||||
self.patchwork_minimum_range_m,
|
||||
self.patchwork_maximum_range_m,
|
||||
)
|
||||
if (
|
||||
not self.profile_id
|
||||
or not all(math.isfinite(value) for value in values)
|
||||
or self.base_link_height_above_ground_m <= 0
|
||||
or self.lidar_height_above_base_link_m <= 0
|
||||
or not math.isclose(
|
||||
self.base_link_height_above_ground_m + self.lidar_height_above_base_link_m,
|
||||
self.patchwork_sensor_height_proxy_m,
|
||||
abs_tol=1e-9,
|
||||
)
|
||||
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
|
||||
or self.patchwork_height_evidence != "published-dimensioned-schematic"
|
||||
):
|
||||
raise ValueError("GOOSE Patchwork++ profile is invalid")
|
||||
|
||||
def to_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": GOOSE_PATCHWORK_PROFILE_SCHEMA,
|
||||
"profile_id": self.profile_id,
|
||||
"source_id": "goose-3d/v2025-08-22",
|
||||
"representation": "native-scan",
|
||||
"sensor_frame": {
|
||||
"frame_id": "sensor/lidar/vls128_roof",
|
||||
"handedness": "right",
|
||||
"x": "forward",
|
||||
"y": "left",
|
||||
"z": "up",
|
||||
"one_revolution": True,
|
||||
},
|
||||
"height": {
|
||||
"base_link_above_ground_m": self.base_link_height_above_ground_m,
|
||||
"lidar_above_base_link_m": self.lidar_height_above_base_link_m,
|
||||
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
|
||||
"evidence": self.patchwork_height_evidence,
|
||||
},
|
||||
"patchworkpp": {
|
||||
"provider_id": "patchworkpp/v1.4.1",
|
||||
"minimum_range_m": self.patchwork_minimum_range_m,
|
||||
"maximum_range_m": self.patchwork_maximum_range_m,
|
||||
"enable_rnr": True,
|
||||
"enable_rvpf": True,
|
||||
"enable_tgr": True,
|
||||
},
|
||||
"evidence": {
|
||||
"dimensioned_schematic": {
|
||||
"url": GOOSE_PAPER_URL,
|
||||
"sha256": GOOSE_PAPER_SHA256,
|
||||
"paper_version": "arxiv-2310.16788v2",
|
||||
"figure": 3,
|
||||
},
|
||||
"tf_topology": {
|
||||
"url": GOOSE_TF_DOT_URL,
|
||||
"sha256": GOOSE_TF_DOT_SHA256,
|
||||
"numeric_transform_present": False,
|
||||
},
|
||||
"first_frame_cross_check": {
|
||||
"method": "independent-ground-near-field-plane",
|
||||
"expected_ground_z_m": -self.patchwork_sensor_height_proxy_m,
|
||||
"observed_intercept_range_m": [-2.18, -2.14],
|
||||
"used_for_calibration": False,
|
||||
},
|
||||
},
|
||||
"scope": {
|
||||
"patchworkpp_eligible": True,
|
||||
"normalized_scan_produced": False,
|
||||
"complete_vehicle_transform_known": False,
|
||||
"deskew_claimed": False,
|
||||
},
|
||||
"authority": {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
DEFAULT_GOOSE_PATCHWORK_PROFILE: Final = GoosePatchworkProfile()
|
||||
|
|
@ -3,17 +3,23 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib
|
||||
import math
|
||||
import platform
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Final, Protocol
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
BoolArray = npt.NDArray[np.bool_]
|
||||
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
|
||||
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
|
||||
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||
|
||||
|
||||
class GroundSegmentationError(ValueError):
|
||||
|
|
@ -142,6 +148,114 @@ class GroundSegmenter(Protocol):
|
|||
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ...
|
||||
|
||||
|
||||
class PatchworkGroundProfile(Protocol):
|
||||
@property
|
||||
def patchwork_sensor_height_proxy_m(self) -> float: ...
|
||||
|
||||
@property
|
||||
def patchwork_minimum_range_m(self) -> float: ...
|
||||
|
||||
@property
|
||||
def patchwork_maximum_range_m(self) -> float: ...
|
||||
|
||||
|
||||
class PatchworkPPGroundSegmenter:
|
||||
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
module: ModuleType,
|
||||
profile: PatchworkGroundProfile,
|
||||
*,
|
||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||
) -> None:
|
||||
if len(source_commit) != 40 or any(
|
||||
character not in "0123456789abcdef" for character in source_commit
|
||||
):
|
||||
raise GroundSegmentationError("Patchwork++ source commit is invalid")
|
||||
if source_tag != PATCHWORKPP_SOURCE_TAG:
|
||||
raise GroundSegmentationError("Patchwork++ source tag is not admitted")
|
||||
module_path_value = getattr(module, "__file__", None)
|
||||
if not isinstance(module_path_value, str):
|
||||
raise GroundSegmentationError("Patchwork++ module has no verifiable binary")
|
||||
module_path = Path(module_path_value).resolve(strict=True)
|
||||
params = module.Parameters()
|
||||
params.sensor_height = profile.patchwork_sensor_height_proxy_m
|
||||
params.min_range = profile.patchwork_minimum_range_m
|
||||
params.max_range = profile.patchwork_maximum_range_m
|
||||
params.enable_RNR = True
|
||||
params.enable_RVPF = True
|
||||
params.enable_TGR = True
|
||||
params.verbose = False
|
||||
self._estimator = module.patchworkpp(params)
|
||||
self._identity = {
|
||||
"provider_id": "patchworkpp/v1.4.1",
|
||||
"source_url": PATCHWORKPP_SOURCE_URL,
|
||||
"source_tag": source_tag,
|
||||
"source_commit": source_commit,
|
||||
"binding_version": str(getattr(module, "__version__", "unknown")),
|
||||
"binary_sha256": _sha256(module_path),
|
||||
"platform": platform.system().lower(),
|
||||
"machine": platform.machine().lower(),
|
||||
"ground_truth": False,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls,
|
||||
profile: PatchworkGroundProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
*,
|
||||
module_name: str = "pypatchworkpp",
|
||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||
) -> PatchworkPPGroundSegmenter:
|
||||
try:
|
||||
module = importlib.import_module(module_name)
|
||||
except ImportError as exc:
|
||||
raise GroundSegmentationError(
|
||||
"Pinned Patchwork++ Python binding is unavailable"
|
||||
) from exc
|
||||
return cls(
|
||||
module,
|
||||
profile,
|
||||
source_commit=source_commit,
|
||||
source_tag=source_tag,
|
||||
)
|
||||
|
||||
@property
|
||||
def identity(self) -> Mapping[str, object]:
|
||||
return self._identity
|
||||
|
||||
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
|
||||
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
|
||||
started = time.perf_counter_ns()
|
||||
self._estimator.estimateGround(points)
|
||||
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
|
||||
ground_indices = np.asarray(
|
||||
self._estimator.getGroundIndices(),
|
||||
dtype=np.int64,
|
||||
).reshape((-1,))
|
||||
nonground_indices = np.asarray(
|
||||
self._estimator.getNongroundIndices(),
|
||||
dtype=np.int64,
|
||||
).reshape((-1,))
|
||||
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
|
||||
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
|
||||
if np.intersect1d(ground_indices, nonground_indices).size:
|
||||
raise GroundSegmentationError("Patchwork++ assigned one point twice")
|
||||
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
||||
assigned = np.zeros(points.shape[0], dtype=np.bool_)
|
||||
ground[ground_indices] = True
|
||||
assigned[ground_indices] = True
|
||||
assigned[nonground_indices] = True
|
||||
return GroundSegmentation(
|
||||
ground_mask=ground,
|
||||
assigned_mask=assigned,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
|
||||
|
||||
class LocalPercentileGroundSegmenter:
|
||||
"""Full-frame diagnostic extension of the existing E19 local ground heuristic."""
|
||||
|
||||
|
|
@ -167,8 +281,7 @@ class LocalPercentileGroundSegmenter:
|
|||
counts = np.bincount(inverse, minlength=unique_cells.shape[0])
|
||||
offsets = np.concatenate(([0], np.cumsum(counts)))
|
||||
cell_lookup = {
|
||||
(int(cell[0]), int(cell[1])): cell_index
|
||||
for cell_index, cell in enumerate(unique_cells)
|
||||
(int(cell[0]), int(cell[1])): cell_index for cell_index, cell in enumerate(unique_cells)
|
||||
}
|
||||
neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1
|
||||
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
||||
|
|
@ -189,9 +302,7 @@ class LocalPercentileGroundSegmenter:
|
|||
if neighbor_cell_index is None:
|
||||
continue
|
||||
neighbor_slices.append(
|
||||
order[
|
||||
offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]
|
||||
]
|
||||
order[offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]]
|
||||
)
|
||||
local_indices = np.concatenate(neighbor_slices)
|
||||
local_xyz = xyz[local_indices]
|
||||
|
|
@ -236,3 +347,10 @@ def _sha256(path: Path) -> str:
|
|||
for chunk in iter(lambda: source.read(1024**2), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _indices(value: npt.NDArray[np.int64], count: int, label: str) -> None:
|
||||
if value.ndim != 1 or np.any(value < 0) or np.any(value >= count):
|
||||
raise GroundSegmentationError(f"{label} indices are invalid")
|
||||
if np.unique(value).size != value.size:
|
||||
raise GroundSegmentationError(f"{label} indices are duplicated")
|
||||
|
|
|
|||
|
|
@ -21,7 +21,11 @@ from k1link.datasets import (
|
|||
read_semantic_kitti_frame,
|
||||
)
|
||||
from k1link.datasets.goose_admission import admit_goose_validation
|
||||
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
|
||||
from k1link.datasets.goose_benchmark import (
|
||||
benchmark_goose_current_ground,
|
||||
benchmark_goose_patchwork_ground,
|
||||
)
|
||||
from k1link.ground_segmentation import GroundSegmentation
|
||||
from k1link.web.lidar_api import build_lidar_router
|
||||
|
||||
|
||||
|
|
@ -354,12 +358,49 @@ def test_goose_current_ground_is_scored_against_independent_labels(
|
|||
assert report["ground_truth"]["evaluated_points"] == 4
|
||||
assert 0 <= report["metrics"]["ground_iou"] <= 1
|
||||
assert report["decision"]["promoted"] is False
|
||||
output = (
|
||||
install
|
||||
/ "benchmarks"
|
||||
/ f"current-ground-{report['identity_sha256']}"
|
||||
/ "preview.json"
|
||||
)
|
||||
preview = read_dataset_ground_preview(output)
|
||||
output = install / "benchmarks" / f"current-ground-{report['identity_sha256']}" / "preview.json"
|
||||
preview = json.loads(output.read_text(encoding="utf-8"))
|
||||
assert preview["point_count"] == 4
|
||||
assert len(preview["disagreement"]) == 4
|
||||
|
||||
class FakePatchwork:
|
||||
identity = {
|
||||
"provider_id": "patchworkpp/v1.4.1",
|
||||
"source_url": "https://github.com/url-kaist/patchwork-plusplus",
|
||||
"source_tag": "v1.4.1",
|
||||
"source_commit": "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||
"binding_version": "test",
|
||||
"binary_sha256": "c" * 64,
|
||||
"platform": "linux",
|
||||
"machine": "x86_64",
|
||||
"ground_truth": False,
|
||||
}
|
||||
|
||||
def segment(self, xyzi: np.ndarray) -> GroundSegmentation:
|
||||
return GroundSegmentation(
|
||||
ground_mask=np.asarray([True, True, False, False]),
|
||||
assigned_mask=np.ones(xyzi.shape[0], dtype=np.bool_),
|
||||
latency_ms=0.2,
|
||||
)
|
||||
|
||||
ab_report = benchmark_goose_patchwork_ground(
|
||||
root,
|
||||
patchwork=FakePatchwork(),
|
||||
preview_points=4,
|
||||
)
|
||||
|
||||
assert ab_report["scope"]["patchworkpp_input_admitted"] is True
|
||||
assert ab_report["scope"]["normalized_scan_produced"] is False
|
||||
assert ab_report["decision"]["promoted"] is False
|
||||
ab_output = (
|
||||
install / "benchmarks" / f"ground-ab-{ab_report['identity_sha256']}" / "preview.json"
|
||||
)
|
||||
ab_preview = read_dataset_ground_preview(ab_output)
|
||||
assert ab_preview["schema_version"] == "missioncore.dataset-ground-comparison-preview/v2"
|
||||
assert ab_preview["input_profile"]["height"]["sensor_above_ground_m"] == 2.24
|
||||
assert len(ab_preview["patchwork_disagreement"]) == 4
|
||||
|
||||
ab_preview["input_profile"]["height"]["sensor_above_ground_m"] = 2.25
|
||||
ab_output.write_text(json.dumps(ab_preview), encoding="utf-8")
|
||||
with pytest.raises(DatasetAdmissionError, match="input profile"):
|
||||
read_dataset_ground_preview(ab_output)
|
||||
|
|
|
|||
Loading…
Reference in New Issue