feat(lidar): qualify Patchwork++ on GOOSE
This commit is contained in:
parent
951b40c870
commit
60ba64004b
|
|
@ -90,25 +90,43 @@ export interface DatasetNativeScanPreview {
|
||||||
}>;
|
}>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DatasetGroundMetrics {
|
||||||
|
precision: number;
|
||||||
|
recall: number;
|
||||||
|
f1: number;
|
||||||
|
groundIou: number;
|
||||||
|
accuracy: number;
|
||||||
|
artificialGroundRecall: number;
|
||||||
|
naturalGroundRecall: number;
|
||||||
|
obstacleNonGroundRecall: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DatasetGroundComparison {
|
export interface DatasetGroundComparison {
|
||||||
sourceId: "goose-3d/v2025-08-22";
|
sourceId: "goose-3d/v2025-08-22";
|
||||||
frameId: string;
|
frameId: string;
|
||||||
pointCount: number;
|
pointCount: number;
|
||||||
currentGround: number[];
|
currentGround: number[];
|
||||||
|
patchworkGround: number[];
|
||||||
|
patchworkAssigned: number[];
|
||||||
groundTruthGround: number[];
|
groundTruthGround: number[];
|
||||||
evaluated: number[];
|
evaluated: number[];
|
||||||
disagreement: number[];
|
currentDisagreement: number[];
|
||||||
|
patchworkDisagreement: number[];
|
||||||
metrics: {
|
metrics: {
|
||||||
precision: number;
|
current: DatasetGroundMetrics;
|
||||||
recall: number;
|
patchworkpp: DatasetGroundMetrics;
|
||||||
f1: number;
|
};
|
||||||
groundIou: number;
|
latencyMs: {
|
||||||
accuracy: number;
|
current: number;
|
||||||
artificialGroundRecall: number;
|
patchworkpp: number;
|
||||||
naturalGroundRecall: number;
|
};
|
||||||
obstacleNonGroundRecall: number;
|
patchworkAssignedFraction: number;
|
||||||
|
inputProfile: {
|
||||||
|
frameId: "sensor/lidar/vls128_roof";
|
||||||
|
sensorHeightM: number;
|
||||||
|
heightEvidence: "published-dimensioned-schematic";
|
||||||
|
normalizedScanProduced: false;
|
||||||
};
|
};
|
||||||
latencyMs: number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DatasetGatewayContractError extends Error {}
|
export class DatasetGatewayContractError extends Error {}
|
||||||
|
|
@ -500,7 +518,7 @@ export function parseDatasetGroundComparison(
|
||||||
): DatasetGroundComparison {
|
): DatasetGroundComparison {
|
||||||
const source = record(value, "Ground comparison");
|
const source = record(value, "Ground comparison");
|
||||||
if (
|
if (
|
||||||
source.schema_version !== "missioncore.dataset-ground-comparison-preview/v1"
|
source.schema_version !== "missioncore.dataset-ground-comparison-preview/v2"
|
||||||
|| source.source_id !== "goose-3d/v2025-08-22"
|
|| source.source_id !== "goose-3d/v2025-08-22"
|
||||||
|| source.sampling !== "deterministic-even-index"
|
|| source.sampling !== "deterministic-even-index"
|
||||||
) {
|
) {
|
||||||
|
|
@ -508,41 +526,135 @@ export function parseDatasetGroundComparison(
|
||||||
}
|
}
|
||||||
const pointCount = number(source.point_count, "point_count");
|
const pointCount = number(source.point_count, "point_count");
|
||||||
const currentGround = integers(source.current_ground, "current_ground", 1);
|
const currentGround = integers(source.current_ground, "current_ground", 1);
|
||||||
|
const patchworkGround = integers(source.patchwork_ground, "patchwork_ground", 1);
|
||||||
|
const patchworkAssigned = integers(
|
||||||
|
source.patchwork_assigned,
|
||||||
|
"patchwork_assigned",
|
||||||
|
1,
|
||||||
|
);
|
||||||
const groundTruthGround = integers(
|
const groundTruthGround = integers(
|
||||||
source.ground_truth_ground,
|
source.ground_truth_ground,
|
||||||
"ground_truth_ground",
|
"ground_truth_ground",
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
const evaluated = integers(source.evaluated, "evaluated", 1);
|
const evaluated = integers(source.evaluated, "evaluated", 1);
|
||||||
const disagreement = integers(source.disagreement, "disagreement", 1);
|
const currentDisagreement = integers(
|
||||||
|
source.current_disagreement,
|
||||||
|
"current_disagreement",
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
const patchworkDisagreement = integers(
|
||||||
|
source.patchwork_disagreement,
|
||||||
|
"patchwork_disagreement",
|
||||||
|
1,
|
||||||
|
);
|
||||||
if (
|
if (
|
||||||
pointCount < 1
|
pointCount < 1
|
||||||
|| pointCount > 50_000
|
|| pointCount > 50_000
|
||||||
|| currentGround.length !== pointCount
|
|| currentGround.length !== pointCount
|
||||||
|
|| patchworkGround.length !== pointCount
|
||||||
|
|| patchworkAssigned.length !== pointCount
|
||||||
|| groundTruthGround.length !== pointCount
|
|| groundTruthGround.length !== pointCount
|
||||||
|| evaluated.length !== pointCount
|
|| evaluated.length !== pointCount
|
||||||
|| disagreement.length !== pointCount
|
|| currentDisagreement.length !== pointCount
|
||||||
|
|| patchworkDisagreement.length !== pointCount
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Ground comparison arrays не выровнены");
|
throw new DatasetGatewayContractError("Ground comparison arrays не выровнены");
|
||||||
}
|
}
|
||||||
const metrics = record(source.metrics, "metrics");
|
const metrics = record(source.metrics, "metrics");
|
||||||
const fraction = (key: string): number => {
|
const parseMetrics = (
|
||||||
const value = number(metrics[key], `metrics.${key}`);
|
value: unknown,
|
||||||
if (value > 1) {
|
label: string,
|
||||||
throw new DatasetGatewayContractError(`metrics.${key}: ожидалась доля`);
|
): DatasetGroundMetrics => {
|
||||||
}
|
const providerMetrics = record(value, label);
|
||||||
return value;
|
const fraction = (key: string): number => {
|
||||||
|
const result = number(providerMetrics[key], `${label}.${key}`);
|
||||||
|
if (result > 1) {
|
||||||
|
throw new DatasetGatewayContractError(`${label}.${key}: ожидалась доля`);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
precision: fraction("precision"),
|
||||||
|
recall: fraction("recall"),
|
||||||
|
f1: fraction("f1"),
|
||||||
|
groundIou: fraction("ground_iou"),
|
||||||
|
accuracy: fraction("accuracy"),
|
||||||
|
artificialGroundRecall: fraction("artificial_ground_recall"),
|
||||||
|
naturalGroundRecall: fraction("natural_ground_recall"),
|
||||||
|
obstacleNonGroundRecall: fraction("obstacle_non_ground_recall"),
|
||||||
|
};
|
||||||
};
|
};
|
||||||
const provider = record(source.provider, "provider");
|
const currentMetrics = parseMetrics(metrics.current, "metrics.current");
|
||||||
|
const patchworkMetrics = parseMetrics(
|
||||||
|
metrics.patchworkpp,
|
||||||
|
"metrics.patchworkpp",
|
||||||
|
);
|
||||||
|
const providers = record(source.providers, "providers");
|
||||||
|
const currentProvider = record(providers.current, "providers.current");
|
||||||
|
const patchworkProvider = record(providers.patchworkpp, "providers.patchworkpp");
|
||||||
if (
|
if (
|
||||||
provider.provider_id !== "missioncore-local-percentile-ground/v1"
|
currentProvider.provider_id !== "missioncore-local-percentile-ground/v1"
|
||||||
|| provider.ground_truth !== false
|
|| currentProvider.ground_truth !== false
|
||||||
|| !/^[a-f0-9]{64}$/.test(
|
|| !/^[a-f0-9]{64}$/.test(
|
||||||
string(provider.implementation_sha256, "provider.implementation_sha256"),
|
string(
|
||||||
|
currentProvider.implementation_sha256,
|
||||||
|
"providers.current.implementation_sha256",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|| patchworkProvider.provider_id !== "patchworkpp/v1.4.1"
|
||||||
|
|| patchworkProvider.source_tag !== "v1.4.1"
|
||||||
|
|| patchworkProvider.source_commit
|
||||||
|
!== "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||||
|
|| patchworkProvider.ground_truth !== false
|
||||||
|
|| !/^[a-f0-9]{64}$/.test(
|
||||||
|
string(
|
||||||
|
patchworkProvider.binary_sha256,
|
||||||
|
"providers.patchworkpp.binary_sha256",
|
||||||
|
),
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("Ground comparison provider несовместим");
|
throw new DatasetGatewayContractError("Ground comparison provider несовместим");
|
||||||
}
|
}
|
||||||
|
const latency = record(source.latency_ms, "latency_ms");
|
||||||
|
const inputProfile = record(source.input_profile, "input_profile");
|
||||||
|
const sensorFrame = record(inputProfile.sensor_frame, "input_profile.sensor_frame");
|
||||||
|
const height = record(inputProfile.height, "input_profile.height");
|
||||||
|
const scope = record(inputProfile.scope, "input_profile.scope");
|
||||||
|
const sensorHeightM = number(
|
||||||
|
height.sensor_above_ground_m,
|
||||||
|
"height.sensor_above_ground_m",
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
inputProfile.schema_version !== "missioncore.goose-patchwork-profile/v1"
|
||||||
|
|| inputProfile.source_id !== "goose-3d/v2025-08-22"
|
||||||
|
|| inputProfile.representation !== "native-scan"
|
||||||
|
|| sensorFrame.frame_id !== "sensor/lidar/vls128_roof"
|
||||||
|
|| sensorFrame.handedness !== "right"
|
||||||
|
|| sensorFrame.x !== "forward"
|
||||||
|
|| sensorFrame.y !== "left"
|
||||||
|
|| sensorFrame.z !== "up"
|
||||||
|
|| sensorFrame.one_revolution !== true
|
||||||
|
|| height.base_link_above_ground_m !== 0.64
|
||||||
|
|| height.lidar_above_base_link_m !== 1.6
|
||||||
|
|| sensorHeightM !== 2.24
|
||||||
|
|| height.evidence !== "published-dimensioned-schematic"
|
||||||
|
|| scope.patchworkpp_eligible !== true
|
||||||
|
|| scope.normalized_scan_produced !== false
|
||||||
|
|| scope.complete_vehicle_transform_known !== false
|
||||||
|
|| scope.deskew_claimed !== false
|
||||||
|
) {
|
||||||
|
throw new DatasetGatewayContractError("GOOSE Patchwork++ profile несовместим");
|
||||||
|
}
|
||||||
|
const patchworkAssignedFraction = number(
|
||||||
|
source.patchwork_assigned_fraction,
|
||||||
|
"patchwork_assigned_fraction",
|
||||||
|
);
|
||||||
|
if (patchworkAssignedFraction > 1) {
|
||||||
|
throw new DatasetGatewayContractError(
|
||||||
|
"patchwork_assigned_fraction: ожидалась доля",
|
||||||
|
);
|
||||||
|
}
|
||||||
const safety = record(source.safety, "safety");
|
const safety = record(source.safety, "safety");
|
||||||
if (
|
if (
|
||||||
safety.qualification_only !== true
|
safety.qualification_only !== true
|
||||||
|
|
@ -555,20 +667,27 @@ export function parseDatasetGroundComparison(
|
||||||
frameId: string(source.frame_id, "frame_id", true),
|
frameId: string(source.frame_id, "frame_id", true),
|
||||||
pointCount,
|
pointCount,
|
||||||
currentGround,
|
currentGround,
|
||||||
|
patchworkGround,
|
||||||
|
patchworkAssigned,
|
||||||
groundTruthGround,
|
groundTruthGround,
|
||||||
evaluated,
|
evaluated,
|
||||||
disagreement,
|
currentDisagreement,
|
||||||
|
patchworkDisagreement,
|
||||||
metrics: {
|
metrics: {
|
||||||
precision: fraction("precision"),
|
current: currentMetrics,
|
||||||
recall: fraction("recall"),
|
patchworkpp: patchworkMetrics,
|
||||||
f1: fraction("f1"),
|
},
|
||||||
groundIou: fraction("ground_iou"),
|
latencyMs: {
|
||||||
accuracy: fraction("accuracy"),
|
current: number(latency.current, "latency_ms.current"),
|
||||||
artificialGroundRecall: fraction("artificial_ground_recall"),
|
patchworkpp: number(latency.patchworkpp, "latency_ms.patchworkpp"),
|
||||||
naturalGroundRecall: fraction("natural_ground_recall"),
|
},
|
||||||
obstacleNonGroundRecall: fraction("obstacle_non_ground_recall"),
|
patchworkAssignedFraction,
|
||||||
|
inputProfile: {
|
||||||
|
frameId: "sensor/lidar/vls128_roof",
|
||||||
|
sensorHeightM,
|
||||||
|
heightEvidence: "published-dimensioned-schematic",
|
||||||
|
normalizedScanProduced: false,
|
||||||
},
|
},
|
||||||
latencyMs: number(source.latency_ms, "latency_ms"),
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -138,9 +138,11 @@ export function DatasetGatewayWorkspace() {
|
||||||
masks: {
|
masks: {
|
||||||
currentGround: comparisonAligned?.currentGround ?? emptyMask,
|
currentGround: comparisonAligned?.currentGround ?? emptyMask,
|
||||||
currentAssigned: comparisonAligned?.evaluated ?? emptyMask,
|
currentAssigned: comparisonAligned?.evaluated ?? emptyMask,
|
||||||
candidateGround: comparisonAligned?.groundTruthGround ?? emptyMask,
|
candidateGround: comparisonAligned?.patchworkGround ?? emptyMask,
|
||||||
candidateAssigned: comparisonAligned?.evaluated ?? emptyMask,
|
candidateAssigned: comparisonAligned?.patchworkAssigned ?? emptyMask,
|
||||||
disagreement: comparisonAligned?.disagreement ?? emptyMask,
|
disagreement: comparisonAligned?.currentDisagreement ?? emptyMask,
|
||||||
|
candidateDisagreement:
|
||||||
|
comparisonAligned?.patchworkDisagreement ?? emptyMask,
|
||||||
groundTruthGround: preview.groundTruthGround,
|
groundTruthGround: preview.groundTruthGround,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
@ -301,7 +303,9 @@ export function DatasetGatewayWorkspace() {
|
||||||
...(comparison
|
...(comparison
|
||||||
? ([
|
? ([
|
||||||
["current", "Current"],
|
["current", "Current"],
|
||||||
["disagreement", "Ошибки"],
|
["candidate", "Patchwork++"],
|
||||||
|
["disagreement", "Ошибки Current"],
|
||||||
|
["candidate-disagreement", "Ошибки PW++"],
|
||||||
] as const)
|
] as const)
|
||||||
: []),
|
: []),
|
||||||
["intensity", "Remission"],
|
["intensity", "Remission"],
|
||||||
|
|
@ -320,20 +324,42 @@ export function DatasetGatewayWorkspace() {
|
||||||
{comparison ? (
|
{comparison ? (
|
||||||
<dl className="dataset-preview__metrics">
|
<dl className="dataset-preview__metrics">
|
||||||
<div>
|
<div>
|
||||||
<dt>Ground IoU</dt>
|
<dt>Current · IoU</dt>
|
||||||
<dd>{(comparison.metrics.groundIou * 100).toFixed(1)}%</dd>
|
<dd>{(comparison.metrics.current.groundIou * 100).toFixed(1)}%</dd>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<dt>Precision</dt>
|
<dt>Current · Recall</dt>
|
||||||
<dd>{(comparison.metrics.precision * 100).toFixed(1)}%</dd>
|
<dd>{(comparison.metrics.current.recall * 100).toFixed(1)}%</dd>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<dt>Recall</dt>
|
<dt>Current · Natural</dt>
|
||||||
<dd>{(comparison.metrics.recall * 100).toFixed(1)}%</dd>
|
<dd>
|
||||||
|
{(comparison.metrics.current.naturalGroundRecall * 100).toFixed(1)}%
|
||||||
|
</dd>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<dt>Latency</dt>
|
<dt>Current · Latency</dt>
|
||||||
<dd>{comparison.latencyMs.toFixed(0)} ms</dd>
|
<dd>{comparison.latencyMs.current.toFixed(0)} ms</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Patchwork++ · IoU</dt>
|
||||||
|
<dd>
|
||||||
|
{(comparison.metrics.patchworkpp.groundIou * 100).toFixed(1)}%
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Patchwork++ · Recall</dt>
|
||||||
|
<dd>{(comparison.metrics.patchworkpp.recall * 100).toFixed(1)}%</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Patchwork++ · Natural</dt>
|
||||||
|
<dd>
|
||||||
|
{(comparison.metrics.patchworkpp.naturalGroundRecall * 100).toFixed(1)}%
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<dt>Patchwork++ · Latency</dt>
|
||||||
|
<dd>{comparison.latencyMs.patchworkpp.toFixed(1)} ms</dd>
|
||||||
</div>
|
</div>
|
||||||
</dl>
|
</dl>
|
||||||
) : comparisonError ? (
|
) : comparisonError ? (
|
||||||
|
|
@ -362,9 +388,19 @@ export function DatasetGatewayWorkspace() {
|
||||||
<span><i className="dataset-legend-ground" />current ground</span>
|
<span><i className="dataset-legend-ground" />current ground</span>
|
||||||
<span><i className="dataset-legend-other" />current non-ground</span>
|
<span><i className="dataset-legend-other" />current non-ground</span>
|
||||||
</>
|
</>
|
||||||
|
) : previewMode === "candidate" ? (
|
||||||
|
<>
|
||||||
|
<span><i className="dataset-legend-ground" />Patchwork++ ground</span>
|
||||||
|
<span><i className="dataset-legend-other" />Patchwork++ non-ground</span>
|
||||||
|
</>
|
||||||
) : previewMode === "disagreement" ? (
|
) : previewMode === "disagreement" ? (
|
||||||
<>
|
<>
|
||||||
<span><i className="dataset-legend-error" />ошибка относительно labels</span>
|
<span><i className="dataset-legend-error" />ошибка Current</span>
|
||||||
|
<span><i className="dataset-legend-other" />совпадение с labels</span>
|
||||||
|
</>
|
||||||
|
) : previewMode === "candidate-disagreement" ? (
|
||||||
|
<>
|
||||||
|
<span><i className="dataset-legend-error" />ошибка Patchwork++</span>
|
||||||
<span><i className="dataset-legend-other" />совпадение</span>
|
<span><i className="dataset-legend-other" />совпадение</span>
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
|
|
@ -380,6 +416,14 @@ export function DatasetGatewayWorkspace() {
|
||||||
<p>{previewError ?? "Проверяем point alignment и разметку."}</p>
|
<p>{previewError ?? "Проверяем point alignment и разметку."}</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
{comparison ? (
|
||||||
|
<p className="dataset-preview__comparison-note">
|
||||||
|
Patchwork++ использует опубликованную высоту VLS-128{" "}
|
||||||
|
{comparison.inputProfile.sensorHeightM.toFixed(2)} м; это
|
||||||
|
algorithm-specific admission, а не полный vehicle TF и не
|
||||||
|
normalized scan.
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
</section>
|
</section>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ export type LidarGroundViewMode =
|
||||||
| "current"
|
| "current"
|
||||||
| "candidate"
|
| "candidate"
|
||||||
| "disagreement"
|
| "disagreement"
|
||||||
|
| "candidate-disagreement"
|
||||||
| "semantic"
|
| "semantic"
|
||||||
| "ground-truth";
|
| "ground-truth";
|
||||||
|
|
||||||
|
|
@ -21,6 +22,7 @@ export interface LidarGroundPointCloudFrame {
|
||||||
candidateGround: number[];
|
candidateGround: number[];
|
||||||
candidateAssigned: number[];
|
candidateAssigned: number[];
|
||||||
disagreement: number[];
|
disagreement: number[];
|
||||||
|
candidateDisagreement?: number[];
|
||||||
groundTruthGround?: number[];
|
groundTruthGround?: number[];
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -119,6 +121,15 @@ function frameColors(
|
||||||
disagreement ? 0.31 : 0.29,
|
disagreement ? 0.31 : 0.29,
|
||||||
disagreement ? 0.22 : 0.35,
|
disagreement ? 0.22 : 0.35,
|
||||||
);
|
);
|
||||||
|
} else if (mode === "candidate-disagreement") {
|
||||||
|
const disagreement = frame.masks.candidateDisagreement?.[index] === 1;
|
||||||
|
setRgb(
|
||||||
|
colors,
|
||||||
|
offset,
|
||||||
|
disagreement ? 1 : 0.24,
|
||||||
|
disagreement ? 0.31 : 0.29,
|
||||||
|
disagreement ? 0.22 : 0.35,
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
setRgb(colors, offset, candidate ? 0.24 : 0.29, candidate ? 0.84 : 0.36, 0.43);
|
setRgb(colors, offset, candidate ? 0.24 : 0.29, candidate ? 0.84 : 0.36, 0.43);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -194,36 +194,97 @@ test("decodes one bounded point-aligned native-scan preview", async () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("decodes a point-aligned current-vs-ground-truth comparison", async () => {
|
test("decodes a point-aligned current-vs-Patchwork++ comparison", async () => {
|
||||||
const payload = {
|
const payload = {
|
||||||
schema_version: "missioncore.dataset-ground-comparison-preview/v1",
|
schema_version: "missioncore.dataset-ground-comparison-preview/v2",
|
||||||
source_id: "goose-3d/v2025-08-22",
|
source_id: "goose-3d/v2025-08-22",
|
||||||
frame_id: "frame-1",
|
frame_id: "frame-1",
|
||||||
sampling: "deterministic-even-index",
|
sampling: "deterministic-even-index",
|
||||||
point_count: 2,
|
point_count: 2,
|
||||||
current_ground: [1, 1],
|
current_ground: [1, 1],
|
||||||
|
patchwork_ground: [1, 0],
|
||||||
|
patchwork_assigned: [1, 1],
|
||||||
ground_truth_ground: [1, 0],
|
ground_truth_ground: [1, 0],
|
||||||
evaluated: [1, 1],
|
evaluated: [1, 1],
|
||||||
disagreement: [0, 1],
|
current_disagreement: [0, 1],
|
||||||
|
patchwork_disagreement: [0, 0],
|
||||||
metrics: {
|
metrics: {
|
||||||
true_positive: 1,
|
current: {
|
||||||
false_positive: 1,
|
true_positive: 1,
|
||||||
false_negative: 0,
|
false_positive: 1,
|
||||||
true_negative: 0,
|
false_negative: 0,
|
||||||
precision: 0.5,
|
true_negative: 0,
|
||||||
recall: 1,
|
precision: 0.5,
|
||||||
f1: 2 / 3,
|
recall: 1,
|
||||||
ground_iou: 0.5,
|
f1: 2 / 3,
|
||||||
accuracy: 0.5,
|
ground_iou: 0.5,
|
||||||
artificial_ground_recall: 1,
|
accuracy: 0.5,
|
||||||
natural_ground_recall: 0,
|
artificial_ground_recall: 1,
|
||||||
obstacle_non_ground_recall: 1,
|
natural_ground_recall: 0,
|
||||||
|
obstacle_non_ground_recall: 1,
|
||||||
|
},
|
||||||
|
patchworkpp: {
|
||||||
|
true_positive: 1,
|
||||||
|
false_positive: 0,
|
||||||
|
false_negative: 0,
|
||||||
|
true_negative: 1,
|
||||||
|
precision: 1,
|
||||||
|
recall: 1,
|
||||||
|
f1: 1,
|
||||||
|
ground_iou: 1,
|
||||||
|
accuracy: 1,
|
||||||
|
artificial_ground_recall: 1,
|
||||||
|
natural_ground_recall: 1,
|
||||||
|
obstacle_non_ground_recall: 1,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
latency_ms: 12.5,
|
latency_ms: {
|
||||||
provider: {
|
current: 12.5,
|
||||||
provider_id: "missioncore-local-percentile-ground/v1",
|
patchworkpp: 0.4,
|
||||||
implementation_sha256: "a".repeat(64),
|
},
|
||||||
ground_truth: false,
|
patchwork_assigned_fraction: 1,
|
||||||
|
providers: {
|
||||||
|
current: {
|
||||||
|
provider_id: "missioncore-local-percentile-ground/v1",
|
||||||
|
implementation_sha256: "a".repeat(64),
|
||||||
|
ground_truth: false,
|
||||||
|
},
|
||||||
|
patchworkpp: {
|
||||||
|
provider_id: "patchworkpp/v1.4.1",
|
||||||
|
source_url: "https://github.com/url-kaist/patchwork-plusplus",
|
||||||
|
source_tag: "v1.4.1",
|
||||||
|
source_commit: "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||||
|
binding_version: "0.0.1",
|
||||||
|
binary_sha256: "b".repeat(64),
|
||||||
|
platform: "linux",
|
||||||
|
machine: "x86_64",
|
||||||
|
ground_truth: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
input_profile: {
|
||||||
|
schema_version: "missioncore.goose-patchwork-profile/v1",
|
||||||
|
source_id: "goose-3d/v2025-08-22",
|
||||||
|
representation: "native-scan",
|
||||||
|
sensor_frame: {
|
||||||
|
frame_id: "sensor/lidar/vls128_roof",
|
||||||
|
handedness: "right",
|
||||||
|
x: "forward",
|
||||||
|
y: "left",
|
||||||
|
z: "up",
|
||||||
|
one_revolution: true,
|
||||||
|
},
|
||||||
|
height: {
|
||||||
|
base_link_above_ground_m: 0.64,
|
||||||
|
lidar_above_base_link_m: 1.6,
|
||||||
|
sensor_above_ground_m: 2.24,
|
||||||
|
evidence: "published-dimensioned-schematic",
|
||||||
|
},
|
||||||
|
scope: {
|
||||||
|
patchworkpp_eligible: true,
|
||||||
|
normalized_scan_produced: false,
|
||||||
|
complete_vehicle_transform_known: false,
|
||||||
|
deskew_claimed: false,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
safety: {
|
safety: {
|
||||||
qualification_only: true,
|
qualification_only: true,
|
||||||
|
|
@ -231,15 +292,23 @@ test("decodes a point-aligned current-vs-ground-truth comparison", async () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
const parsed = parseDatasetGroundComparison(payload);
|
const parsed = parseDatasetGroundComparison(payload);
|
||||||
assert.equal(parsed.metrics.groundIou, 0.5);
|
assert.equal(parsed.metrics.current.groundIou, 0.5);
|
||||||
assert.deepEqual(parsed.disagreement, [0, 1]);
|
assert.equal(parsed.metrics.patchworkpp.groundIou, 1);
|
||||||
|
assert.deepEqual(parsed.patchworkDisagreement, [0, 0]);
|
||||||
|
assert.equal(parsed.inputProfile.sensorHeightM, 2.24);
|
||||||
|
|
||||||
const fetched = await fetchDatasetGroundComparison({
|
const fetched = await fetchDatasetGroundComparison({
|
||||||
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
fetcher: async () => new Response(JSON.stringify(payload), { status: 200 }),
|
||||||
});
|
});
|
||||||
assert.equal(fetched.latencyMs, 12.5);
|
assert.equal(fetched.latencyMs.patchworkpp, 0.4);
|
||||||
|
|
||||||
payload.disagreement.pop();
|
payload.patchwork_disagreement.pop();
|
||||||
|
assert.throws(
|
||||||
|
() => parseDatasetGroundComparison(payload),
|
||||||
|
DatasetGatewayContractError,
|
||||||
|
);
|
||||||
|
payload.patchwork_disagreement.push(0);
|
||||||
|
payload.input_profile.height.sensor_above_ground_m = 2.25;
|
||||||
assert.throws(
|
assert.throws(
|
||||||
() => parseDatasetGroundComparison(payload),
|
() => parseDatasetGroundComparison(payload),
|
||||||
DatasetGatewayContractError,
|
DatasetGatewayContractError,
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
Date: 2026-07-25
|
Date: 2026-07-25
|
||||||
Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete;
|
Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete;
|
||||||
Dataset Gateway S0 implemented, first real GOOSE import pending worker D storage
|
Dataset Gateway first-frame Current/Patchwork++ A/B complete
|
||||||
Scope: real scanner records, replay and future live shadow processing
|
Scope: real scanner records, replay and future live shadow processing
|
||||||
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
||||||
|
|
||||||
|
|
@ -293,7 +293,7 @@ path. K1 manual review or a new real vehicle dataset is reserved for later
|
||||||
domain adaptation after a public baseline proves that the pipeline and metric
|
domain adaptation after a public baseline proves that the pipeline and metric
|
||||||
harness work.
|
harness work.
|
||||||
|
|
||||||
### L2.5 — Dataset Gateway — S0 complete, real import pending
|
### L2.5 — Dataset Gateway — first-frame A/B complete
|
||||||
|
|
||||||
- [x] Define separate `native-scan`, `normalized-scan` and
|
- [x] Define separate `native-scan`, `normalized-scan` and
|
||||||
`rolling-local-map` representations.
|
`rolling-local-map` representations.
|
||||||
|
|
@ -304,13 +304,16 @@ harness work.
|
||||||
per-point time and line/ring fields.
|
per-point time and line/ring fields.
|
||||||
- [x] Require operator-admitted storage under
|
- [x] Require operator-admitted storage under
|
||||||
`D:\NDC_MISSIONCORE\datasets`.
|
`D:\NDC_MISSIONCORE\datasets`.
|
||||||
- [ ] Configure the worker dataset root and record disk/resource baseline.
|
- [x] Configure the worker dataset root and record disk/resource baseline.
|
||||||
- [ ] Download only the 3.3 GB GOOSE validation archive first and record its
|
- [x] Download only the 3.3 GB GOOSE validation archive first and record its
|
||||||
hash/license/provenance.
|
hash/license/provenance.
|
||||||
- [ ] Show one real labeled revolution in React with native remission and
|
- [x] Show one real labeled revolution in React with native remission and
|
||||||
ground-truth superclass coloring.
|
ground-truth superclass coloring.
|
||||||
- [ ] Admit an explicit frame/mounting profile before normalization.
|
- [x] Admit the published Patchwork-specific axes/height profile without
|
||||||
- [ ] Run current ground and Patchwork++ against GOOSE ground truth.
|
claiming full vehicle TF, deskew or a general normalized scan.
|
||||||
|
- [x] Run current ground and Patchwork++ against the same GOOSE native scan and
|
||||||
|
ground truth; keep the result one-frame diagnostic.
|
||||||
|
- [ ] Expand Current/Patchwork++ qualification to the validation split.
|
||||||
- [ ] Add named range/FOV/density/noise/dropout degradation profiles without
|
- [ ] Add named range/FOV/density/noise/dropout degradation profiles without
|
||||||
overwriting the native frame.
|
overwriting the native frame.
|
||||||
|
|
||||||
|
|
@ -387,8 +390,8 @@ The near-term value is not a prettier point cloud:
|
||||||
- hardware selection becomes evidence-driven: a future vehicle LiDAR is
|
- hardware selection becomes evidence-driven: a future vehicle LiDAR is
|
||||||
accepted by its timing/fields/profile, not by vendor marketing.
|
accepted by its timing/fields/profile, not by vendor marketing.
|
||||||
|
|
||||||
The highest-value immediate work is the worker-side GOOSE validation import,
|
The highest-value immediate work is now the validation-split
|
||||||
the first labeled native scan in React and an accuracy-bearing ground A/B.
|
Current/Patchwork++ gate plus named range/FOV/density/noise/dropout degradation
|
||||||
LiDAR-native detection follows on the same gateway. Nvblox and alternative
|
profiles. LiDAR-native detection follows on the same gateway. Nvblox and
|
||||||
SLAM remain later because their timing, pose and scan-geometry gates are not yet
|
alternative SLAM remain later because their timing, pose and scan-geometry
|
||||||
satisfied.
|
gates are not yet satisfied.
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,12 @@ Implemented now:
|
||||||
`Открыть` action;
|
`Открыть` action;
|
||||||
- one admitted GOOSE validation frame with source colors, normalized remission
|
- one admitted GOOSE validation frame with source colors, normalized remission
|
||||||
and an independent ground-truth view;
|
and an independent ground-truth view;
|
||||||
|
- a published, dimensioned MuCAR-3/VLS-128 Patchwork++ input profile:
|
||||||
|
sensor frame `x-forward / y-left / z-up`, physical height `2.24 m`, native
|
||||||
|
one-revolution scan and explicit absence of deskew/full vehicle TF claims;
|
||||||
|
- a reproducible Current/Patchwork++ A/B artifact with point-aligned masks,
|
||||||
|
disagreement views, independent-label metrics, latency and exact provider
|
||||||
|
identities;
|
||||||
- a separate **Парк → Диагностика LiDAR** surface containing only real sensor
|
- a separate **Парк → Диагностика LiDAR** surface containing only real sensor
|
||||||
recordings and their operational evidence.
|
recordings and their operational evidence.
|
||||||
|
|
||||||
|
|
@ -152,11 +158,13 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
||||||
4. [Done] Import one labeled frame and expose it in React as `native-scan`.
|
4. [Done] Import one labeled frame and expose it in React as `native-scan`.
|
||||||
5. [Done] Show native remission, the original semantic palette and
|
5. [Done] Show native remission, the original semantic palette and
|
||||||
ground-truth superclass coloring.
|
ground-truth superclass coloring.
|
||||||
6. Add a declared GOOSE frame/mounting profile and produce `normalized-scan`.
|
6. [Patchwork-specific profile done; general normalization still pending] Admit
|
||||||
7. [Current baseline done; Patchwork++ blocked on mounting evidence] Run the
|
the published VLS-128 axes and physical height without claiming a complete
|
||||||
current ground heuristic and Patchwork++ against independent labels.
|
numeric vehicle TF or deskewed `normalized-scan`.
|
||||||
|
7. [One-frame A/B done] Run the current ground heuristic and pinned
|
||||||
|
Patchwork++ against the same native scan and independent labels.
|
||||||
8. Add sensor-degradation profiles for range, FOV, density, noise and dropout.
|
8. Add sensor-degradation profiles for range, FOV, density, noise and dropout.
|
||||||
9. Only after the one-frame contract passes, expand to the validation split and
|
9. Expand the A/B to the validation split; only after that qualification gate,
|
||||||
add a rolling-map sequence with localization evidence.
|
add a rolling-map sequence with localization evidence.
|
||||||
|
|
||||||
## Acceptance checklist
|
## Acceptance checklist
|
||||||
|
|
@ -172,9 +180,11 @@ time; TTL/dynamic filtering prevents stale ghosts.
|
||||||
- [x] Worker D root configured.
|
- [x] Worker D root configured.
|
||||||
- [x] GOOSE validation archive hash recorded.
|
- [x] GOOSE validation archive hash recorded.
|
||||||
- [x] First real labeled frame visible in React.
|
- [x] First real labeled frame visible in React.
|
||||||
- [ ] Coordinate and mounting profile admitted.
|
- [x] Patchwork-specific axes and physical-height profile admitted.
|
||||||
|
- [ ] Complete vehicle transform and general `normalized-scan` admitted.
|
||||||
- [x] Current local-percentile baseline measured against ground truth.
|
- [x] Current local-percentile baseline measured against ground truth.
|
||||||
- [ ] Patchwork++ accuracy measured against ground truth.
|
- [x] Patchwork++ one-frame accuracy measured against ground truth.
|
||||||
|
- [ ] Current/Patchwork++ validation-split gate qualified.
|
||||||
- [ ] Sensor-degradation matrix qualified.
|
- [ ] Sensor-degradation matrix qualified.
|
||||||
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
- [ ] Rolling local map with pose/TTL/dynamic policy qualified.
|
||||||
|
|
||||||
|
|
@ -229,10 +239,54 @@ previous exact-radius result. This removes the previous all-points scan for
|
||||||
every occupied cell, but the measured latency still classifies it as a
|
every occupied cell, but the measured latency still classifies it as a
|
||||||
diagnostic baseline rather than an onboard candidate.
|
diagnostic baseline rather than an onboard candidate.
|
||||||
|
|
||||||
Patchwork++ is intentionally not scored yet. The validation ZIP contains XYZI,
|
## First Patchwork++ A/B result
|
||||||
labels, mapping, LICENSE and CHANGELOG but no numeric TF/mounting calibration.
|
|
||||||
GOOSE documents the VLS-128 as a roof LiDAR and publishes a separate MuCAR-3 TF
|
GOOSE's dimensioned MuCAR-3 schematic provides the two vertical dimensions
|
||||||
tree, but the graph image alone is not physical-height evidence. The next gate
|
needed by this algorithm-specific gate: `base_link` is `0.64 m` above ground
|
||||||
is to admit the numeric transform from `base_link_ground` to
|
and the VLS-128 optical center is `1.60 m` above `base_link`. The admitted
|
||||||
`sensor/lidar/vls128_roof`, declare the source axis convention, and only then
|
Patchwork++ height is therefore `2.24 m`. The same published schematic declares
|
||||||
run Patchwork++.
|
the sensor axes as `x` forward, `y` left and `z` up. An independent fit to
|
||||||
|
GOOSE-labeled near-field ground observed a `-2.18 .. -2.14 m` intercept; this
|
||||||
|
was a non-calibrating cross-check, not the source of the height.
|
||||||
|
|
||||||
|
This narrowly admits the input required by Patchwork++ on the native
|
||||||
|
sensor-centric scan. It does **not** claim a complete numeric vehicle transform,
|
||||||
|
per-point timing, deskew or a general `normalized-scan`.
|
||||||
|
|
||||||
|
Official Patchwork++ `v1.4.1`, source commit
|
||||||
|
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`, was run against the same first
|
||||||
|
frame and independent labels as Current:
|
||||||
|
|
||||||
|
| Metric | Current | Patchwork++ |
|
||||||
|
| --- | ---: | ---: |
|
||||||
|
| Ground IoU | `49.6165%` | `60.3702%` |
|
||||||
|
| Precision | `73.5892%` | `72.6612%` |
|
||||||
|
| Recall | `60.3659%` | `78.1130%` |
|
||||||
|
| F1 | `66.3249%` | `75.2886%` |
|
||||||
|
| Artificial-ground recall | `95.2364%` | `98.9688%` |
|
||||||
|
| Natural-ground recall | `49.4516%` | `71.5853%` |
|
||||||
|
| Obstacle non-ground recall | `79.6145%` | `92.1729%` |
|
||||||
|
| Worker latency | `3966.53 ms` | `15.89 ms` |
|
||||||
|
|
||||||
|
Reproducibility pins:
|
||||||
|
|
||||||
|
- benchmark identity:
|
||||||
|
`2a6d05f54a9e2ac727d9c850c1133f2fb539bfeddbd97c07cfd198ccb239162c`;
|
||||||
|
- full point-aligned prediction SHA-256:
|
||||||
|
`190f455c6e47911921b7f6454913e1bb2d0b8b5814ee2e34d7b63295afddc7ef`;
|
||||||
|
- bounded browser preview SHA-256:
|
||||||
|
`e23ca573103faf931523415e6c263248737eb6600f482b63de685b725bbc28c3`;
|
||||||
|
- Patchwork++ binary SHA-256:
|
||||||
|
`be8038b2098c83fe53841aa8ae19e362910e9056fe0ee7b9304c1ee5c5941094`.
|
||||||
|
|
||||||
|
Patchwork++ wins this frame by `10.75` percentage points of Ground IoU, raises
|
||||||
|
natural-ground recall by `22.13` points and is roughly `250x` faster in this
|
||||||
|
run. The result is deliberately `one-frame-diagnostic`: it makes Patchwork++
|
||||||
|
the candidate for validation-split and degradation qualification, not an
|
||||||
|
accepted navigation or safety provider.
|
||||||
|
|
||||||
|
Primary source evidence:
|
||||||
|
|
||||||
|
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/);
|
||||||
|
- [GOOSE paper, Figure 3](https://arxiv.org/pdf/2310.16788);
|
||||||
|
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1).
|
||||||
|
|
|
||||||
|
|
@ -87,24 +87,34 @@ unavailable rather than inventing timestamps.
|
||||||
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
- Sensor adaptation may change range, FOV, point density, noise and dropout for
|
||||||
robustness experiments, but it cannot recreate lost timestamps, occlusions
|
robustness experiments, but it cannot recreate lost timestamps, occlusions
|
||||||
or material response.
|
or material response.
|
||||||
- Patchwork++ becomes eligible for a real quality gate only on a sensor-centric
|
- Patchwork++ becomes eligible for an algorithm-specific quality gate on a
|
||||||
scan with declared scan geometry and physical mounting height plus
|
sensor-centric scan with declared axes, physical mounting height and
|
||||||
independent labels.
|
independent labels. That narrow admission does not itself create a general
|
||||||
|
`normalized-scan`.
|
||||||
- Stable operator visualization is owned by rolling-map policy, not by the
|
- Stable operator visualization is owned by rolling-map policy, not by the
|
||||||
ground classifier.
|
ground classifier.
|
||||||
- Public labels may qualify an algorithm independently of the production
|
- Public labels may qualify an algorithm independently of the production
|
||||||
sensor. The first GOOSE frame exposed a `49.62%` Ground IoU and only `49.45%`
|
sensor. The first GOOSE frame exposed a `49.62%` Ground IoU and only `49.45%`
|
||||||
natural-ground recall for the current local-percentile baseline; these are
|
natural-ground recall for the current local-percentile baseline; these are
|
||||||
diagnostic results, not production promotion.
|
diagnostic results, not production promotion.
|
||||||
- A dataset label contract does not imply a mounting contract. Patchwork++
|
- A dataset label contract does not imply a mounting contract. For MuCAR-3,
|
||||||
remains blocked until the numeric GOOSE roof-LiDAR transform and physical
|
GOOSE's published dimensioned schematic admits the Patchwork-specific
|
||||||
height are admitted from source evidence.
|
`2.24 m` sensor height (`0.64 + 1.60 m`) and `x-forward / y-left / z-up`
|
||||||
|
axes. The complete numeric vehicle transform, deskew and general
|
||||||
|
`normalized-scan` remain unavailable.
|
||||||
|
- On the first independently labeled frame, pinned Patchwork++ `v1.4.1`
|
||||||
|
achieved `60.37%` Ground IoU, `71.59%` natural-ground recall and `15.89 ms`
|
||||||
|
worker latency versus Current's `49.62%`, `49.45%` and `3966.53 ms`.
|
||||||
|
This is a one-frame diagnostic candidate decision, not production promotion.
|
||||||
|
|
||||||
## Primary references
|
## Primary references
|
||||||
|
|
||||||
- [GOOSE dataset structure](https://goose-dataset.de/docs/dataset-structure/)
|
- [GOOSE dataset structure](https://goose-dataset.de/docs/dataset-structure/)
|
||||||
- [GOOSE setup and archive sizes](https://goose-dataset.de/docs/setup/)
|
- [GOOSE setup and archive sizes](https://goose-dataset.de/docs/setup/)
|
||||||
- [GOOSE 3D challenge ontology](https://goose-dataset.de/docs/3d-semantic-segmentation-challenge/)
|
- [GOOSE 3D challenge ontology](https://goose-dataset.de/docs/3d-semantic-segmentation-challenge/)
|
||||||
|
- [GOOSE MuCAR-3 sensor setup](https://goose-dataset.de/docs/mucar3/)
|
||||||
|
- [GOOSE paper and dimensioned sensor schematic](https://arxiv.org/pdf/2310.16788)
|
||||||
|
- [Patchwork++ v1.4.1](https://github.com/url-kaist/patchwork-plusplus/tree/v1.4.1)
|
||||||
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
- [Livox ROS Driver 2 point formats](https://github.com/Livox-SDK/livox_ros_driver2)
|
||||||
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
- [Livox LIO motion-distortion handling](https://github.com/Livox-SDK/LIO-Livox)
|
||||||
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
- [ROS FilterDeskew timestamp requirement](https://docs.ros.org/en/noetic/api/mp2p_icp/html/classmp2p__icp__filters_1_1FilterDeskew.html)
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,14 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import importlib
|
|
||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import platform
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import time
|
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import ModuleType
|
|
||||||
from typing import Any, Final
|
from typing import Any, Final
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
@ -25,9 +21,21 @@ from k1link.ground_segmentation import (
|
||||||
GroundSegmenter,
|
GroundSegmenter,
|
||||||
LocalPercentileGroundSegmenter,
|
LocalPercentileGroundSegmenter,
|
||||||
)
|
)
|
||||||
|
from k1link.ground_segmentation import (
|
||||||
|
PATCHWORKPP_SOURCE_COMMIT as PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
)
|
||||||
|
from k1link.ground_segmentation import (
|
||||||
|
PATCHWORKPP_SOURCE_TAG as PATCHWORKPP_SOURCE_TAG,
|
||||||
|
)
|
||||||
|
from k1link.ground_segmentation import (
|
||||||
|
PATCHWORKPP_SOURCE_URL as PATCHWORKPP_SOURCE_URL,
|
||||||
|
)
|
||||||
from k1link.ground_segmentation import (
|
from k1link.ground_segmentation import (
|
||||||
GroundSegmentationError as LidarGroundError,
|
GroundSegmentationError as LidarGroundError,
|
||||||
)
|
)
|
||||||
|
from k1link.ground_segmentation import (
|
||||||
|
PatchworkPPGroundSegmenter as PatchworkPPGroundSegmenter,
|
||||||
|
)
|
||||||
|
|
||||||
from .lidar_contract import LidarContractError, sensor_frame_xyzi
|
from .lidar_contract import LidarContractError, sensor_frame_xyzi
|
||||||
from .lidar_replay import LidarReplayPackV2
|
from .lidar_replay import LidarReplayPackV2
|
||||||
|
|
@ -42,107 +50,9 @@ LIDAR_GROUND_MANIFEST_NAME: Final = "manifest.json"
|
||||||
LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz"
|
LIDAR_GROUND_ANNOTATION_LABELS_NAME: Final = "labels-template.npz"
|
||||||
MAX_GROUND_FRAME_POINTS: Final = 200_000
|
MAX_GROUND_FRAME_POINTS: Final = 200_000
|
||||||
|
|
||||||
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
|
|
||||||
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
|
|
||||||
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
|
||||||
|
|
||||||
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
|
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
|
||||||
_ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$")
|
_ANNOTATION_TEMPLATE_ID = re.compile(r"^ground-annotation-template-[a-f0-9]{64}$")
|
||||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
_GIT_SHA1 = re.compile(r"^[a-f0-9]{40}$")
|
|
||||||
|
|
||||||
|
|
||||||
class PatchworkPPGroundSegmenter:
|
|
||||||
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
module: ModuleType,
|
|
||||||
profile: GroundBenchmarkProfile,
|
|
||||||
*,
|
|
||||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
|
||||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
|
||||||
) -> None:
|
|
||||||
if _GIT_SHA1.fullmatch(source_commit) is None:
|
|
||||||
raise LidarGroundError("Patchwork++ source commit is invalid")
|
|
||||||
if source_tag != PATCHWORKPP_SOURCE_TAG:
|
|
||||||
raise LidarGroundError("Patchwork++ source tag is not admitted")
|
|
||||||
module_path_value = getattr(module, "__file__", None)
|
|
||||||
if not isinstance(module_path_value, str):
|
|
||||||
raise LidarGroundError("Patchwork++ module has no verifiable binary")
|
|
||||||
module_path = Path(module_path_value).resolve(strict=True)
|
|
||||||
params = module.Parameters()
|
|
||||||
params.sensor_height = profile.patchwork_sensor_height_proxy_m
|
|
||||||
params.min_range = profile.patchwork_minimum_range_m
|
|
||||||
params.max_range = profile.patchwork_maximum_range_m
|
|
||||||
params.enable_RNR = True
|
|
||||||
params.enable_RVPF = True
|
|
||||||
params.enable_TGR = True
|
|
||||||
params.verbose = False
|
|
||||||
self._estimator = module.patchworkpp(params)
|
|
||||||
self._identity = {
|
|
||||||
"provider_id": "patchworkpp/v1.4.1",
|
|
||||||
"source_url": PATCHWORKPP_SOURCE_URL,
|
|
||||||
"source_tag": source_tag,
|
|
||||||
"source_commit": source_commit,
|
|
||||||
"binding_version": str(getattr(module, "__version__", "unknown")),
|
|
||||||
"binary_sha256": _sha256(module_path),
|
|
||||||
"platform": platform.system().lower(),
|
|
||||||
"machine": platform.machine().lower(),
|
|
||||||
"ground_truth": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def load(
|
|
||||||
cls,
|
|
||||||
profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
|
||||||
*,
|
|
||||||
module_name: str = "pypatchworkpp",
|
|
||||||
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
|
||||||
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
|
||||||
) -> PatchworkPPGroundSegmenter:
|
|
||||||
try:
|
|
||||||
module = importlib.import_module(module_name)
|
|
||||||
except ImportError as exc:
|
|
||||||
raise LidarGroundError("Pinned Patchwork++ Python binding is unavailable") from exc
|
|
||||||
return cls(
|
|
||||||
module,
|
|
||||||
profile,
|
|
||||||
source_commit=source_commit,
|
|
||||||
source_tag=source_tag,
|
|
||||||
)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def identity(self) -> Mapping[str, object]:
|
|
||||||
return self._identity
|
|
||||||
|
|
||||||
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
|
|
||||||
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
|
|
||||||
started = time.perf_counter_ns()
|
|
||||||
self._estimator.estimateGround(points)
|
|
||||||
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
|
|
||||||
ground_indices = np.asarray(
|
|
||||||
self._estimator.getGroundIndices(),
|
|
||||||
dtype=np.int64,
|
|
||||||
).reshape((-1,))
|
|
||||||
nonground_indices = np.asarray(
|
|
||||||
self._estimator.getNongroundIndices(),
|
|
||||||
dtype=np.int64,
|
|
||||||
).reshape((-1,))
|
|
||||||
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
|
|
||||||
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
|
|
||||||
if np.intersect1d(ground_indices, nonground_indices).size:
|
|
||||||
raise LidarGroundError("Patchwork++ assigned one point twice")
|
|
||||||
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
|
||||||
assigned = np.zeros(points.shape[0], dtype=np.bool_)
|
|
||||||
ground[ground_indices] = True
|
|
||||||
assigned[ground_indices] = True
|
|
||||||
assigned[nonground_indices] = True
|
|
||||||
return GroundSegmentation(
|
|
||||||
ground_mask=ground,
|
|
||||||
assigned_mask=assigned,
|
|
||||||
latency_ms=latency_ms,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class LidarGroundBenchmarkV1:
|
class LidarGroundBenchmarkV1:
|
||||||
|
|
|
||||||
|
|
@ -15,9 +15,17 @@ from k1link.datasets.gateway import (
|
||||||
read_semantic_kitti_frame,
|
read_semantic_kitti_frame,
|
||||||
)
|
)
|
||||||
from k1link.datasets.goose_benchmark import (
|
from k1link.datasets.goose_benchmark import (
|
||||||
|
GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
|
||||||
|
GOOSE_GROUND_AB_PREVIEW_SCHEMA,
|
||||||
GOOSE_GROUND_BENCHMARK_SCHEMA,
|
GOOSE_GROUND_BENCHMARK_SCHEMA,
|
||||||
GOOSE_GROUND_PREVIEW_SCHEMA,
|
GOOSE_GROUND_PREVIEW_SCHEMA,
|
||||||
benchmark_goose_current_ground,
|
benchmark_goose_current_ground,
|
||||||
|
benchmark_goose_patchwork_ground,
|
||||||
|
)
|
||||||
|
from k1link.datasets.goose_profile import (
|
||||||
|
DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||||
|
GOOSE_PATCHWORK_PROFILE_SCHEMA,
|
||||||
|
GoosePatchworkProfile,
|
||||||
)
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|
@ -25,9 +33,15 @@ __all__ = [
|
||||||
"DatasetAdmissionError",
|
"DatasetAdmissionError",
|
||||||
"DatasetFrameError",
|
"DatasetFrameError",
|
||||||
"DatasetPointFrame",
|
"DatasetPointFrame",
|
||||||
|
"DEFAULT_GOOSE_PATCHWORK_PROFILE",
|
||||||
|
"GOOSE_GROUND_AB_BENCHMARK_SCHEMA",
|
||||||
|
"GOOSE_GROUND_AB_PREVIEW_SCHEMA",
|
||||||
"GOOSE_GROUND_BENCHMARK_SCHEMA",
|
"GOOSE_GROUND_BENCHMARK_SCHEMA",
|
||||||
"GOOSE_GROUND_PREVIEW_SCHEMA",
|
"GOOSE_GROUND_PREVIEW_SCHEMA",
|
||||||
|
"GOOSE_PATCHWORK_PROFILE_SCHEMA",
|
||||||
|
"GoosePatchworkProfile",
|
||||||
"benchmark_goose_current_ground",
|
"benchmark_goose_current_ground",
|
||||||
|
"benchmark_goose_patchwork_ground",
|
||||||
"configured_dataset_admission_manifest",
|
"configured_dataset_admission_manifest",
|
||||||
"configured_dataset_ground_preview",
|
"configured_dataset_ground_preview",
|
||||||
"configured_dataset_preview",
|
"configured_dataset_preview",
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,10 @@ from typing import Annotated
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation
|
from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation
|
||||||
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
|
from k1link.datasets.goose_benchmark import (
|
||||||
|
benchmark_goose_current_ground,
|
||||||
|
benchmark_goose_patchwork_ground,
|
||||||
|
)
|
||||||
|
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
add_completion=False,
|
add_completion=False,
|
||||||
|
|
@ -63,5 +66,22 @@ def benchmark_goose_current_ground_command(
|
||||||
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
@app.command("benchmark-goose-patchwork-ground")
|
||||||
|
def benchmark_goose_patchwork_ground_command(
|
||||||
|
dataset_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
) -> None:
|
||||||
|
"""Compare current and pinned Patchwork++ providers against GOOSE labels."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
report = benchmark_goose_patchwork_ground(dataset_root)
|
||||||
|
except (GooseAdmissionError, ValueError) as exc:
|
||||||
|
typer.echo(str(exc), err=True)
|
||||||
|
raise typer.Exit(code=2) from exc
|
||||||
|
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app()
|
app()
|
||||||
|
|
|
||||||
|
|
@ -16,10 +16,12 @@ from typing import Any, Final, Literal
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import numpy.typing as npt
|
import numpy.typing as npt
|
||||||
|
|
||||||
|
from k1link.datasets.goose_profile import DEFAULT_GOOSE_PATCHWORK_PROFILE
|
||||||
|
|
||||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
||||||
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||||
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||||
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
|
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
|
||||||
DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST"
|
DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST"
|
||||||
DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW"
|
DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW"
|
||||||
|
|
@ -316,8 +318,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
||||||
raise DatasetAdmissionError("dataset admission status is incompatible")
|
raise DatasetAdmissionError("dataset admission status is incompatible")
|
||||||
storage = _object(document["storage"], "storage")
|
storage = _object(document["storage"], "storage")
|
||||||
if (
|
if (
|
||||||
set(storage)
|
set(storage) != {"policy", "admitted", "canonical_root", "path_exposed"}
|
||||||
!= {"policy", "admitted", "canonical_root", "path_exposed"}
|
|
||||||
or storage.get("policy") != "worker-d-only"
|
or storage.get("policy") != "worker-d-only"
|
||||||
or storage.get("admitted") is not True
|
or storage.get("admitted") is not True
|
||||||
or storage.get("canonical_root") is not True
|
or storage.get("canonical_root") is not True
|
||||||
|
|
@ -338,8 +339,7 @@ def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
||||||
"vendor_checksum_available",
|
"vendor_checksum_available",
|
||||||
}
|
}
|
||||||
or archive.get("filename") != "goose_3d_val.zip"
|
or archive.get("filename") != "goose_3d_val.zip"
|
||||||
or archive.get("source_url")
|
or archive.get("source_url") != "https://goose-dataset.de/storage/goose_3d_val.zip"
|
||||||
!= "https://goose-dataset.de/storage/goose_3d_val.zip"
|
|
||||||
or archive.get("vendor_checksum_available") is not False
|
or archive.get("vendor_checksum_available") is not False
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError("dataset archive admission is incompatible")
|
raise DatasetAdmissionError("dataset archive admission is incompatible")
|
||||||
|
|
@ -385,9 +385,7 @@ def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError("dataset preview identity is incompatible")
|
raise DatasetAdmissionError("dataset preview identity is incompatible")
|
||||||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||||
source_point_count = _positive_integer(
|
source_point_count = _positive_integer(document.get("source_point_count"), "source_point_count")
|
||||||
document.get("source_point_count"), "source_point_count"
|
|
||||||
)
|
|
||||||
if point_count > 50_000 or point_count > source_point_count:
|
if point_count > 50_000 or point_count > source_point_count:
|
||||||
raise DatasetAdmissionError("dataset preview point count is incompatible")
|
raise DatasetAdmissionError("dataset preview point count is incompatible")
|
||||||
points = document.get("points_xyz_m")
|
points = document.get("points_xyz_m")
|
||||||
|
|
@ -456,20 +454,28 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
||||||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||||
if point_count > 50_000:
|
if point_count > 50_000:
|
||||||
raise DatasetAdmissionError("dataset ground preview point count is incompatible")
|
raise DatasetAdmissionError("dataset ground preview point count is incompatible")
|
||||||
for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"):
|
for key in (
|
||||||
|
"current_ground",
|
||||||
|
"patchwork_ground",
|
||||||
|
"patchwork_assigned",
|
||||||
|
"ground_truth_ground",
|
||||||
|
"evaluated",
|
||||||
|
"current_disagreement",
|
||||||
|
"patchwork_disagreement",
|
||||||
|
):
|
||||||
values = document.get(key)
|
values = document.get(key)
|
||||||
if (
|
if (
|
||||||
not isinstance(values, list)
|
not isinstance(values, list)
|
||||||
or len(values) != point_count
|
or len(values) != point_count
|
||||||
or any(
|
or any(
|
||||||
not isinstance(value, int)
|
not isinstance(value, int) or isinstance(value, bool) or value not in (0, 1)
|
||||||
or isinstance(value, bool)
|
|
||||||
or value not in (0, 1)
|
|
||||||
for value in values
|
for value in values
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible")
|
raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible")
|
||||||
metrics = _object(document.get("metrics"), "metrics")
|
metrics_by_provider = _object(document.get("metrics"), "metrics")
|
||||||
|
if set(metrics_by_provider) != {"current", "patchworkpp"}:
|
||||||
|
raise DatasetAdmissionError("dataset ground provider metrics are incompatible")
|
||||||
expected_metrics = {
|
expected_metrics = {
|
||||||
"true_positive",
|
"true_positive",
|
||||||
"false_positive",
|
"false_positive",
|
||||||
|
|
@ -484,40 +490,88 @@ def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
||||||
"natural_ground_recall",
|
"natural_ground_recall",
|
||||||
"obstacle_non_ground_recall",
|
"obstacle_non_ground_recall",
|
||||||
}
|
}
|
||||||
if set(metrics) != expected_metrics:
|
for provider_id in ("current", "patchworkpp"):
|
||||||
raise DatasetAdmissionError("dataset ground metrics are incompatible")
|
metrics = _object(metrics_by_provider[provider_id], f"metrics.{provider_id}")
|
||||||
for key, value in metrics.items():
|
if set(metrics) != expected_metrics:
|
||||||
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
|
raise DatasetAdmissionError("dataset ground metrics are incompatible")
|
||||||
_nonnegative_integer(value, f"metrics.{key}")
|
for key, value in metrics.items():
|
||||||
elif (
|
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
|
||||||
|
_nonnegative_integer(value, f"metrics.{provider_id}.{key}")
|
||||||
|
elif (
|
||||||
|
not isinstance(value, (int, float))
|
||||||
|
or isinstance(value, bool)
|
||||||
|
or not np.isfinite(value)
|
||||||
|
or not 0 <= value <= 1
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError(
|
||||||
|
f"dataset ground metric {provider_id}.{key} is incompatible"
|
||||||
|
)
|
||||||
|
latency = _object(document.get("latency_ms"), "latency_ms")
|
||||||
|
if set(latency) != {"current", "patchworkpp"}:
|
||||||
|
raise DatasetAdmissionError("dataset ground latency providers are incompatible")
|
||||||
|
for provider_id, value in latency.items():
|
||||||
|
if (
|
||||||
not isinstance(value, (int, float))
|
not isinstance(value, (int, float))
|
||||||
or isinstance(value, bool)
|
or isinstance(value, bool)
|
||||||
or not np.isfinite(value)
|
or not np.isfinite(value)
|
||||||
or not 0 <= value <= 1
|
or value < 0
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible")
|
raise DatasetAdmissionError(f"dataset ground latency {provider_id} is incompatible")
|
||||||
latency = document.get("latency_ms")
|
providers = _object(document.get("providers"), "providers")
|
||||||
|
if set(providers) != {"current", "patchworkpp"}:
|
||||||
|
raise DatasetAdmissionError("dataset ground providers are incompatible")
|
||||||
|
current_provider = _object(providers["current"], "providers.current")
|
||||||
if (
|
if (
|
||||||
not isinstance(latency, (int, float))
|
set(current_provider) != {"provider_id", "implementation_sha256", "ground_truth"}
|
||||||
or isinstance(latency, bool)
|
or current_provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
|
||||||
or not np.isfinite(latency)
|
or current_provider.get("ground_truth") is not False
|
||||||
or latency < 0
|
|
||||||
):
|
|
||||||
raise DatasetAdmissionError("dataset ground latency is incompatible")
|
|
||||||
provider = _object(document.get("provider"), "provider")
|
|
||||||
if (
|
|
||||||
set(provider) != {"provider_id", "implementation_sha256", "ground_truth"}
|
|
||||||
or provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
|
|
||||||
or provider.get("ground_truth") is not False
|
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError("dataset ground provider is incompatible")
|
raise DatasetAdmissionError("dataset ground provider is incompatible")
|
||||||
digest = provider.get("implementation_sha256")
|
digest = current_provider.get("implementation_sha256")
|
||||||
if (
|
if (
|
||||||
not isinstance(digest, str)
|
not isinstance(digest, str)
|
||||||
or len(digest) != 64
|
or len(digest) != 64
|
||||||
or any(character not in "0123456789abcdef" for character in digest)
|
or any(character not in "0123456789abcdef" for character in digest)
|
||||||
):
|
):
|
||||||
raise DatasetAdmissionError("dataset ground provider digest is incompatible")
|
raise DatasetAdmissionError("dataset ground provider digest is incompatible")
|
||||||
|
patchwork_provider = _object(providers["patchworkpp"], "providers.patchworkpp")
|
||||||
|
if (
|
||||||
|
set(patchwork_provider)
|
||||||
|
!= {
|
||||||
|
"provider_id",
|
||||||
|
"source_url",
|
||||||
|
"source_tag",
|
||||||
|
"source_commit",
|
||||||
|
"binding_version",
|
||||||
|
"binary_sha256",
|
||||||
|
"platform",
|
||||||
|
"machine",
|
||||||
|
"ground_truth",
|
||||||
|
}
|
||||||
|
or patchwork_provider.get("provider_id") != "patchworkpp/v1.4.1"
|
||||||
|
or patchwork_provider.get("source_tag") != "v1.4.1"
|
||||||
|
or patchwork_provider.get("source_commit") != "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||||
|
or patchwork_provider.get("ground_truth") is not False
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("dataset Patchwork++ provider is incompatible")
|
||||||
|
binary_digest = patchwork_provider.get("binary_sha256")
|
||||||
|
if (
|
||||||
|
not isinstance(binary_digest, str)
|
||||||
|
or len(binary_digest) != 64
|
||||||
|
or any(character not in "0123456789abcdef" for character in binary_digest)
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("dataset Patchwork++ binary digest is incompatible")
|
||||||
|
assigned_fraction = document.get("patchwork_assigned_fraction")
|
||||||
|
if (
|
||||||
|
not isinstance(assigned_fraction, (int, float))
|
||||||
|
or isinstance(assigned_fraction, bool)
|
||||||
|
or not np.isfinite(assigned_fraction)
|
||||||
|
or not 0 <= assigned_fraction <= 1
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("dataset Patchwork++ assigned fraction is incompatible")
|
||||||
|
input_profile = _object(document.get("input_profile"), "input_profile")
|
||||||
|
if input_profile != DEFAULT_GOOSE_PATCHWORK_PROFILE.to_dict():
|
||||||
|
raise DatasetAdmissionError("dataset Patchwork++ input profile is incompatible")
|
||||||
safety = _object(document.get("safety"), "safety")
|
safety = _object(document.get("safety"), "safety")
|
||||||
if safety != {
|
if safety != {
|
||||||
"qualification_only": True,
|
"qualification_only": True,
|
||||||
|
|
|
||||||
|
|
@ -18,14 +18,22 @@ from k1link.datasets.goose_admission import (
|
||||||
GooseAdmissionError,
|
GooseAdmissionError,
|
||||||
read_goose_label_mapping,
|
read_goose_label_mapping,
|
||||||
)
|
)
|
||||||
|
from k1link.datasets.goose_profile import (
|
||||||
|
DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||||
|
GoosePatchworkProfile,
|
||||||
|
)
|
||||||
from k1link.ground_segmentation import (
|
from k1link.ground_segmentation import (
|
||||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
GroundBenchmarkProfile,
|
GroundBenchmarkProfile,
|
||||||
|
GroundSegmenter,
|
||||||
LocalPercentileGroundSegmenter,
|
LocalPercentileGroundSegmenter,
|
||||||
|
PatchworkPPGroundSegmenter,
|
||||||
)
|
)
|
||||||
|
|
||||||
GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1"
|
GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1"
|
||||||
GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
||||||
|
GOOSE_GROUND_AB_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-ab-benchmark/v1"
|
||||||
|
GOOSE_GROUND_AB_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v2"
|
||||||
MAX_PREVIEW_POINTS: Final = 50_000
|
MAX_PREVIEW_POINTS: Final = 50_000
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -135,8 +143,7 @@ def benchmark_goose_current_ground(
|
||||||
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
||||||
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
||||||
"disagreement": (
|
"disagreement": (
|
||||||
evaluated[indices]
|
evaluated[indices] & (prediction.ground_mask[indices] != ground_truth[indices])
|
||||||
& (prediction.ground_mask[indices] != ground_truth[indices])
|
|
||||||
)
|
)
|
||||||
.astype(np.uint8)
|
.astype(np.uint8)
|
||||||
.tolist(),
|
.tolist(),
|
||||||
|
|
@ -179,6 +186,204 @@ def benchmark_goose_current_ground(
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def benchmark_goose_patchwork_ground(
|
||||||
|
dataset_root: Path,
|
||||||
|
*,
|
||||||
|
current_profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
|
patchwork_profile: GoosePatchworkProfile = DEFAULT_GOOSE_PATCHWORK_PROFILE,
|
||||||
|
patchwork: GroundSegmenter | None = None,
|
||||||
|
patchwork_module_name: str = "pypatchworkpp",
|
||||||
|
preview_points: int = MAX_PREVIEW_POINTS,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Score current and pinned Patchwork++ providers against one GOOSE frame."""
|
||||||
|
|
||||||
|
root = dataset_root.expanduser().absolute()
|
||||||
|
if not _is_worker_dataset_root(root):
|
||||||
|
raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root")
|
||||||
|
manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json")
|
||||||
|
if manifest["status"] != "frame-ready":
|
||||||
|
raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking")
|
||||||
|
archive_sha256 = str(manifest["archive"]["sha256"])
|
||||||
|
frame_id = str(manifest["frame"]["frame_id"])
|
||||||
|
install = root / "goose-3d/v2025-08-22/installs" / archive_sha256
|
||||||
|
frame_root = install / "frames" / frame_id
|
||||||
|
frame = read_semantic_kitti_frame(
|
||||||
|
frame_root / "points.bin",
|
||||||
|
frame_root / "labels.label",
|
||||||
|
)
|
||||||
|
labels = read_goose_label_mapping(install / "goose_label_mapping.csv")
|
||||||
|
categories = np.asarray(
|
||||||
|
[labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels],
|
||||||
|
dtype=np.uint8,
|
||||||
|
)
|
||||||
|
evaluated = categories != 0
|
||||||
|
ground_truth = (categories == 2) | (categories == 3)
|
||||||
|
xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype(
|
||||||
|
np.float32,
|
||||||
|
copy=False,
|
||||||
|
)
|
||||||
|
current = LocalPercentileGroundSegmenter(current_profile)
|
||||||
|
candidate = (
|
||||||
|
patchwork
|
||||||
|
if patchwork is not None
|
||||||
|
else PatchworkPPGroundSegmenter.load(
|
||||||
|
patchwork_profile,
|
||||||
|
module_name=patchwork_module_name,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
current_result = current.segment(xyzi)
|
||||||
|
candidate_result = candidate.segment(xyzi)
|
||||||
|
for result, label in (
|
||||||
|
(current_result, "current"),
|
||||||
|
(candidate_result, "Patchwork++"),
|
||||||
|
):
|
||||||
|
if result.ground_mask.shape != (frame.point_count,) or result.assigned_mask.shape != (
|
||||||
|
frame.point_count,
|
||||||
|
):
|
||||||
|
raise GooseAdmissionError(f"{label} result is not point-aligned")
|
||||||
|
current_metrics = _ground_metrics(
|
||||||
|
current_result.ground_mask,
|
||||||
|
ground_truth,
|
||||||
|
evaluated,
|
||||||
|
categories,
|
||||||
|
)
|
||||||
|
candidate_metrics = _ground_metrics(
|
||||||
|
candidate_result.ground_mask,
|
||||||
|
ground_truth,
|
||||||
|
evaluated,
|
||||||
|
categories,
|
||||||
|
)
|
||||||
|
candidate_assigned_fraction = float(np.mean(candidate_result.assigned_mask))
|
||||||
|
profile_document = patchwork_profile.to_dict()
|
||||||
|
current_profile_document = _goose_current_profile_document(current_profile)
|
||||||
|
identity_document = {
|
||||||
|
"source_id": GOOSE_SOURCE_ID,
|
||||||
|
"archive_sha256": archive_sha256,
|
||||||
|
"frame_id": frame_id,
|
||||||
|
"source_point_count": frame.point_count,
|
||||||
|
"input_profile": profile_document,
|
||||||
|
"current_profile": current_profile_document,
|
||||||
|
"providers": {
|
||||||
|
"current": dict(current.identity),
|
||||||
|
"patchworkpp": dict(candidate.identity),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
identity_sha256 = _canonical_sha256(identity_document)
|
||||||
|
output_root = install / "benchmarks" / f"ground-ab-{identity_sha256}"
|
||||||
|
output_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
report_path = output_root / "report.json"
|
||||||
|
if report_path.is_file():
|
||||||
|
try:
|
||||||
|
existing = json.loads(report_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise GooseAdmissionError("existing GOOSE A/B report is invalid") from exc
|
||||||
|
if (
|
||||||
|
not isinstance(existing, dict)
|
||||||
|
or existing.get("schema_version") != GOOSE_GROUND_AB_BENCHMARK_SCHEMA
|
||||||
|
or existing.get("identity_sha256") != identity_sha256
|
||||||
|
or not (output_root / "prediction.npz").is_file()
|
||||||
|
or not (output_root / "preview.json").is_file()
|
||||||
|
):
|
||||||
|
raise GooseAdmissionError("existing GOOSE A/B benchmark is incomplete")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
prediction_path = output_root / "prediction.npz"
|
||||||
|
_atomic_npz(
|
||||||
|
prediction_path,
|
||||||
|
current_ground=current_result.ground_mask.astype(np.uint8),
|
||||||
|
patchwork_ground=candidate_result.ground_mask.astype(np.uint8),
|
||||||
|
patchwork_assigned=candidate_result.assigned_mask.astype(np.uint8),
|
||||||
|
ground_truth_ground=ground_truth.astype(np.uint8),
|
||||||
|
evaluated=evaluated.astype(np.uint8),
|
||||||
|
)
|
||||||
|
sample_count = min(frame.point_count, preview_points)
|
||||||
|
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
|
||||||
|
current_disagreement = evaluated & (current_result.ground_mask != ground_truth)
|
||||||
|
candidate_disagreement = evaluated & (candidate_result.ground_mask != ground_truth)
|
||||||
|
preview = {
|
||||||
|
"schema_version": GOOSE_GROUND_AB_PREVIEW_SCHEMA,
|
||||||
|
"source_id": GOOSE_SOURCE_ID,
|
||||||
|
"frame_id": frame_id,
|
||||||
|
"sampling": "deterministic-even-index",
|
||||||
|
"point_count": sample_count,
|
||||||
|
"current_ground": current_result.ground_mask[indices].astype(np.uint8).tolist(),
|
||||||
|
"patchwork_ground": candidate_result.ground_mask[indices].astype(np.uint8).tolist(),
|
||||||
|
"patchwork_assigned": (candidate_result.assigned_mask[indices].astype(np.uint8).tolist()),
|
||||||
|
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
||||||
|
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
||||||
|
"current_disagreement": (current_disagreement[indices].astype(np.uint8).tolist()),
|
||||||
|
"patchwork_disagreement": (candidate_disagreement[indices].astype(np.uint8).tolist()),
|
||||||
|
"metrics": {
|
||||||
|
"current": current_metrics,
|
||||||
|
"patchworkpp": candidate_metrics,
|
||||||
|
},
|
||||||
|
"latency_ms": {
|
||||||
|
"current": current_result.latency_ms,
|
||||||
|
"patchworkpp": candidate_result.latency_ms,
|
||||||
|
},
|
||||||
|
"patchwork_assigned_fraction": candidate_assigned_fraction,
|
||||||
|
"providers": {
|
||||||
|
"current": dict(current.identity),
|
||||||
|
"patchworkpp": dict(candidate.identity),
|
||||||
|
},
|
||||||
|
"input_profile": profile_document,
|
||||||
|
"safety": {
|
||||||
|
"qualification_only": True,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
preview_path = output_root / "preview.json"
|
||||||
|
_atomic_json(preview_path, preview)
|
||||||
|
winner = (
|
||||||
|
"patchworkpp"
|
||||||
|
if candidate_metrics["ground_iou"] > current_metrics["ground_iou"]
|
||||||
|
else "current"
|
||||||
|
if current_metrics["ground_iou"] > candidate_metrics["ground_iou"]
|
||||||
|
else "tie"
|
||||||
|
)
|
||||||
|
report = {
|
||||||
|
"schema_version": GOOSE_GROUND_AB_BENCHMARK_SCHEMA,
|
||||||
|
"identity_sha256": identity_sha256,
|
||||||
|
"identity": identity_document,
|
||||||
|
"ground_truth": {
|
||||||
|
"source": "GOOSE point-wise semantic labels",
|
||||||
|
"ground_categories": ["artificial_ground", "natural_ground"],
|
||||||
|
"void_excluded": True,
|
||||||
|
"evaluated_points": int(np.count_nonzero(evaluated)),
|
||||||
|
},
|
||||||
|
"providers": {
|
||||||
|
"current": {
|
||||||
|
"metrics": current_metrics,
|
||||||
|
"latency_ms": current_result.latency_ms,
|
||||||
|
"assigned_fraction": 1.0,
|
||||||
|
},
|
||||||
|
"patchworkpp": {
|
||||||
|
"metrics": candidate_metrics,
|
||||||
|
"latency_ms": candidate_result.latency_ms,
|
||||||
|
"assigned_fraction": candidate_assigned_fraction,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"artifacts": {
|
||||||
|
"prediction_sha256": _sha256_file(prediction_path),
|
||||||
|
"preview_sha256": _sha256_file(preview_path),
|
||||||
|
},
|
||||||
|
"decision": {
|
||||||
|
"status": "one-frame-diagnostic",
|
||||||
|
"promoted": False,
|
||||||
|
"winner_by_ground_iou": winner,
|
||||||
|
"reason": "one public frame does not qualify a production ground provider",
|
||||||
|
"next_gate": "validation-split qualification with degradation profiles",
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"patchworkpp_input_admitted": True,
|
||||||
|
"normalized_scan_produced": False,
|
||||||
|
"complete_vehicle_transform_known": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_atomic_json(report_path, report)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
def _is_worker_dataset_root(root: Path) -> bool:
|
def _is_worker_dataset_root(root: Path) -> bool:
|
||||||
normalized = str(root).replace("\\", "/").rstrip("/").lower()
|
normalized = str(root).replace("\\", "/").rstrip("/").lower()
|
||||||
return normalized == "/mnt/d/ndc_missioncore/datasets"
|
return normalized == "/mnt/d/ndc_missioncore/datasets"
|
||||||
|
|
@ -216,6 +421,45 @@ def _binary_metrics(
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ground_metrics(
|
||||||
|
prediction: np.ndarray[Any, Any],
|
||||||
|
ground_truth: np.ndarray[Any, Any],
|
||||||
|
evaluated: np.ndarray[Any, Any],
|
||||||
|
categories: np.ndarray[Any, Any],
|
||||||
|
) -> dict[str, float | int]:
|
||||||
|
metrics = _binary_metrics(prediction, ground_truth, evaluated)
|
||||||
|
metrics.update(
|
||||||
|
{
|
||||||
|
"artificial_ground_recall": _recall(prediction, categories == 2),
|
||||||
|
"natural_ground_recall": _recall(prediction, categories == 3),
|
||||||
|
"obstacle_non_ground_recall": _recall(~prediction, categories == 4),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return metrics
|
||||||
|
|
||||||
|
|
||||||
|
def _goose_current_profile_document(
|
||||||
|
profile: GroundBenchmarkProfile,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
parameters = profile.to_dict()["current_baseline"]
|
||||||
|
return {
|
||||||
|
"schema_version": "missioncore.goose-current-ground-profile/v1",
|
||||||
|
"profile_id": "goose-native-local-percentile/v1",
|
||||||
|
"source_id": GOOSE_SOURCE_ID,
|
||||||
|
"representation": "native-scan",
|
||||||
|
"sensor_frame": "sensor/lidar/vls128_roof",
|
||||||
|
"provider": parameters,
|
||||||
|
"scope": {
|
||||||
|
"complete_vehicle_transform_required": False,
|
||||||
|
"normalized_scan_produced": False,
|
||||||
|
},
|
||||||
|
"authority": {
|
||||||
|
"qualification_only": True,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float:
|
def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float:
|
||||||
return _ratio(
|
return _ratio(
|
||||||
int(np.count_nonzero(prediction & truth)),
|
int(np.count_nonzero(prediction & truth)),
|
||||||
|
|
@ -259,9 +503,7 @@ def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None:
|
||||||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||||
if path.exists():
|
if path.exists():
|
||||||
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
|
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
|
||||||
encoded = (
|
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
|
||||||
)
|
|
||||||
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
|
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
|
||||||
temporary_path = Path(temporary.name)
|
temporary_path = Path(temporary.name)
|
||||||
temporary.write(encoded)
|
temporary.write(encoded)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,117 @@
|
||||||
|
"""Published, algorithm-scoped GOOSE VLS-128 ground profile."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
GOOSE_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.goose-patchwork-profile/v1"
|
||||||
|
GOOSE_PAPER_URL: Final = "https://arxiv.org/pdf/2310.16788"
|
||||||
|
GOOSE_PAPER_SHA256: Final = "cb17f38c3ae498917966a63bc0e034cc5dd3fa4ad35873b121592f99026cc77e"
|
||||||
|
GOOSE_TF_DOT_URL: Final = "https://goose-dataset.de/docs/resources/mucar3_tf/mucar3_tf.dot"
|
||||||
|
GOOSE_TF_DOT_SHA256: Final = "1b10e905bcff17b90305ff02734b5e6bb7f5753d9b8e74cd7c43f3d0f32ef469"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class GoosePatchworkProfile:
|
||||||
|
"""Narrow admission profile for one native GOOSE scan.
|
||||||
|
|
||||||
|
Figure 3 of the official paper dimensions ``base_link``/INS at 0.64 m
|
||||||
|
above ground and the VLS-128 optical center 1.60 m above ``base_link``.
|
||||||
|
Their sum is the physical height Patchwork++ requires. This does not claim
|
||||||
|
a complete vehicle mounting transform and cannot produce a normalized scan.
|
||||||
|
"""
|
||||||
|
|
||||||
|
profile_id: str = "goose-mucar3-vls128-patchwork/v1"
|
||||||
|
base_link_height_above_ground_m: float = 0.64
|
||||||
|
lidar_height_above_base_link_m: float = 1.60
|
||||||
|
patchwork_sensor_height_proxy_m: float = 2.24
|
||||||
|
patchwork_minimum_range_m: float = 2.7
|
||||||
|
patchwork_maximum_range_m: float = 80.0
|
||||||
|
patchwork_height_evidence: str = "published-dimensioned-schematic"
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
values = (
|
||||||
|
self.base_link_height_above_ground_m,
|
||||||
|
self.lidar_height_above_base_link_m,
|
||||||
|
self.patchwork_sensor_height_proxy_m,
|
||||||
|
self.patchwork_minimum_range_m,
|
||||||
|
self.patchwork_maximum_range_m,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not self.profile_id
|
||||||
|
or not all(math.isfinite(value) for value in values)
|
||||||
|
or self.base_link_height_above_ground_m <= 0
|
||||||
|
or self.lidar_height_above_base_link_m <= 0
|
||||||
|
or not math.isclose(
|
||||||
|
self.base_link_height_above_ground_m + self.lidar_height_above_base_link_m,
|
||||||
|
self.patchwork_sensor_height_proxy_m,
|
||||||
|
abs_tol=1e-9,
|
||||||
|
)
|
||||||
|
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
|
||||||
|
or self.patchwork_height_evidence != "published-dimensioned-schematic"
|
||||||
|
):
|
||||||
|
raise ValueError("GOOSE Patchwork++ profile is invalid")
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"schema_version": GOOSE_PATCHWORK_PROFILE_SCHEMA,
|
||||||
|
"profile_id": self.profile_id,
|
||||||
|
"source_id": "goose-3d/v2025-08-22",
|
||||||
|
"representation": "native-scan",
|
||||||
|
"sensor_frame": {
|
||||||
|
"frame_id": "sensor/lidar/vls128_roof",
|
||||||
|
"handedness": "right",
|
||||||
|
"x": "forward",
|
||||||
|
"y": "left",
|
||||||
|
"z": "up",
|
||||||
|
"one_revolution": True,
|
||||||
|
},
|
||||||
|
"height": {
|
||||||
|
"base_link_above_ground_m": self.base_link_height_above_ground_m,
|
||||||
|
"lidar_above_base_link_m": self.lidar_height_above_base_link_m,
|
||||||
|
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
|
||||||
|
"evidence": self.patchwork_height_evidence,
|
||||||
|
},
|
||||||
|
"patchworkpp": {
|
||||||
|
"provider_id": "patchworkpp/v1.4.1",
|
||||||
|
"minimum_range_m": self.patchwork_minimum_range_m,
|
||||||
|
"maximum_range_m": self.patchwork_maximum_range_m,
|
||||||
|
"enable_rnr": True,
|
||||||
|
"enable_rvpf": True,
|
||||||
|
"enable_tgr": True,
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"dimensioned_schematic": {
|
||||||
|
"url": GOOSE_PAPER_URL,
|
||||||
|
"sha256": GOOSE_PAPER_SHA256,
|
||||||
|
"paper_version": "arxiv-2310.16788v2",
|
||||||
|
"figure": 3,
|
||||||
|
},
|
||||||
|
"tf_topology": {
|
||||||
|
"url": GOOSE_TF_DOT_URL,
|
||||||
|
"sha256": GOOSE_TF_DOT_SHA256,
|
||||||
|
"numeric_transform_present": False,
|
||||||
|
},
|
||||||
|
"first_frame_cross_check": {
|
||||||
|
"method": "independent-ground-near-field-plane",
|
||||||
|
"expected_ground_z_m": -self.patchwork_sensor_height_proxy_m,
|
||||||
|
"observed_intercept_range_m": [-2.18, -2.14],
|
||||||
|
"used_for_calibration": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"patchworkpp_eligible": True,
|
||||||
|
"normalized_scan_produced": False,
|
||||||
|
"complete_vehicle_transform_known": False,
|
||||||
|
"deskew_claimed": False,
|
||||||
|
},
|
||||||
|
"authority": {
|
||||||
|
"qualification_only": True,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_GOOSE_PATCHWORK_PROFILE: Final = GoosePatchworkProfile()
|
||||||
|
|
@ -3,17 +3,23 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import importlib
|
||||||
import math
|
import math
|
||||||
|
import platform
|
||||||
import time
|
import time
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from types import ModuleType
|
||||||
from typing import Final, Protocol
|
from typing import Final, Protocol
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import numpy.typing as npt
|
import numpy.typing as npt
|
||||||
|
|
||||||
BoolArray = npt.NDArray[np.bool_]
|
BoolArray = npt.NDArray[np.bool_]
|
||||||
|
PATCHWORKPP_SOURCE_URL: Final = "https://github.com/url-kaist/patchwork-plusplus"
|
||||||
|
PATCHWORKPP_SOURCE_TAG: Final = "v1.4.1"
|
||||||
|
PATCHWORKPP_SOURCE_COMMIT: Final = "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c"
|
||||||
|
|
||||||
|
|
||||||
class GroundSegmentationError(ValueError):
|
class GroundSegmentationError(ValueError):
|
||||||
|
|
@ -142,6 +148,114 @@ class GroundSegmenter(Protocol):
|
||||||
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ...
|
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation: ...
|
||||||
|
|
||||||
|
|
||||||
|
class PatchworkGroundProfile(Protocol):
|
||||||
|
@property
|
||||||
|
def patchwork_sensor_height_proxy_m(self) -> float: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def patchwork_minimum_range_m(self) -> float: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def patchwork_maximum_range_m(self) -> float: ...
|
||||||
|
|
||||||
|
|
||||||
|
class PatchworkPPGroundSegmenter:
|
||||||
|
"""Runtime-only adapter for the pinned official Patchwork++ Python binding."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
module: ModuleType,
|
||||||
|
profile: PatchworkGroundProfile,
|
||||||
|
*,
|
||||||
|
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||||
|
) -> None:
|
||||||
|
if len(source_commit) != 40 or any(
|
||||||
|
character not in "0123456789abcdef" for character in source_commit
|
||||||
|
):
|
||||||
|
raise GroundSegmentationError("Patchwork++ source commit is invalid")
|
||||||
|
if source_tag != PATCHWORKPP_SOURCE_TAG:
|
||||||
|
raise GroundSegmentationError("Patchwork++ source tag is not admitted")
|
||||||
|
module_path_value = getattr(module, "__file__", None)
|
||||||
|
if not isinstance(module_path_value, str):
|
||||||
|
raise GroundSegmentationError("Patchwork++ module has no verifiable binary")
|
||||||
|
module_path = Path(module_path_value).resolve(strict=True)
|
||||||
|
params = module.Parameters()
|
||||||
|
params.sensor_height = profile.patchwork_sensor_height_proxy_m
|
||||||
|
params.min_range = profile.patchwork_minimum_range_m
|
||||||
|
params.max_range = profile.patchwork_maximum_range_m
|
||||||
|
params.enable_RNR = True
|
||||||
|
params.enable_RVPF = True
|
||||||
|
params.enable_TGR = True
|
||||||
|
params.verbose = False
|
||||||
|
self._estimator = module.patchworkpp(params)
|
||||||
|
self._identity = {
|
||||||
|
"provider_id": "patchworkpp/v1.4.1",
|
||||||
|
"source_url": PATCHWORKPP_SOURCE_URL,
|
||||||
|
"source_tag": source_tag,
|
||||||
|
"source_commit": source_commit,
|
||||||
|
"binding_version": str(getattr(module, "__version__", "unknown")),
|
||||||
|
"binary_sha256": _sha256(module_path),
|
||||||
|
"platform": platform.system().lower(),
|
||||||
|
"machine": platform.machine().lower(),
|
||||||
|
"ground_truth": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(
|
||||||
|
cls,
|
||||||
|
profile: PatchworkGroundProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
|
*,
|
||||||
|
module_name: str = "pypatchworkpp",
|
||||||
|
source_commit: str = PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
source_tag: str = PATCHWORKPP_SOURCE_TAG,
|
||||||
|
) -> PatchworkPPGroundSegmenter:
|
||||||
|
try:
|
||||||
|
module = importlib.import_module(module_name)
|
||||||
|
except ImportError as exc:
|
||||||
|
raise GroundSegmentationError(
|
||||||
|
"Pinned Patchwork++ Python binding is unavailable"
|
||||||
|
) from exc
|
||||||
|
return cls(
|
||||||
|
module,
|
||||||
|
profile,
|
||||||
|
source_commit=source_commit,
|
||||||
|
source_tag=source_tag,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def identity(self) -> Mapping[str, object]:
|
||||||
|
return self._identity
|
||||||
|
|
||||||
|
def segment(self, xyzi: npt.NDArray[np.float32]) -> GroundSegmentation:
|
||||||
|
points = np.ascontiguousarray(_xyzi(xyzi), dtype=np.float32)
|
||||||
|
started = time.perf_counter_ns()
|
||||||
|
self._estimator.estimateGround(points)
|
||||||
|
latency_ms = (time.perf_counter_ns() - started) / 1_000_000
|
||||||
|
ground_indices = np.asarray(
|
||||||
|
self._estimator.getGroundIndices(),
|
||||||
|
dtype=np.int64,
|
||||||
|
).reshape((-1,))
|
||||||
|
nonground_indices = np.asarray(
|
||||||
|
self._estimator.getNongroundIndices(),
|
||||||
|
dtype=np.int64,
|
||||||
|
).reshape((-1,))
|
||||||
|
_indices(ground_indices, points.shape[0], "Patchwork++ ground")
|
||||||
|
_indices(nonground_indices, points.shape[0], "Patchwork++ non-ground")
|
||||||
|
if np.intersect1d(ground_indices, nonground_indices).size:
|
||||||
|
raise GroundSegmentationError("Patchwork++ assigned one point twice")
|
||||||
|
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
||||||
|
assigned = np.zeros(points.shape[0], dtype=np.bool_)
|
||||||
|
ground[ground_indices] = True
|
||||||
|
assigned[ground_indices] = True
|
||||||
|
assigned[nonground_indices] = True
|
||||||
|
return GroundSegmentation(
|
||||||
|
ground_mask=ground,
|
||||||
|
assigned_mask=assigned,
|
||||||
|
latency_ms=latency_ms,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class LocalPercentileGroundSegmenter:
|
class LocalPercentileGroundSegmenter:
|
||||||
"""Full-frame diagnostic extension of the existing E19 local ground heuristic."""
|
"""Full-frame diagnostic extension of the existing E19 local ground heuristic."""
|
||||||
|
|
||||||
|
|
@ -167,8 +281,7 @@ class LocalPercentileGroundSegmenter:
|
||||||
counts = np.bincount(inverse, minlength=unique_cells.shape[0])
|
counts = np.bincount(inverse, minlength=unique_cells.shape[0])
|
||||||
offsets = np.concatenate(([0], np.cumsum(counts)))
|
offsets = np.concatenate(([0], np.cumsum(counts)))
|
||||||
cell_lookup = {
|
cell_lookup = {
|
||||||
(int(cell[0]), int(cell[1])): cell_index
|
(int(cell[0]), int(cell[1])): cell_index for cell_index, cell in enumerate(unique_cells)
|
||||||
for cell_index, cell in enumerate(unique_cells)
|
|
||||||
}
|
}
|
||||||
neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1
|
neighbor_span = math.ceil(self.profile.current_local_radius_m / cell_size) + 1
|
||||||
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
ground = np.zeros(points.shape[0], dtype=np.bool_)
|
||||||
|
|
@ -189,9 +302,7 @@ class LocalPercentileGroundSegmenter:
|
||||||
if neighbor_cell_index is None:
|
if neighbor_cell_index is None:
|
||||||
continue
|
continue
|
||||||
neighbor_slices.append(
|
neighbor_slices.append(
|
||||||
order[
|
order[offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]]
|
||||||
offsets[neighbor_cell_index] : offsets[neighbor_cell_index + 1]
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
local_indices = np.concatenate(neighbor_slices)
|
local_indices = np.concatenate(neighbor_slices)
|
||||||
local_xyz = xyz[local_indices]
|
local_xyz = xyz[local_indices]
|
||||||
|
|
@ -236,3 +347,10 @@ def _sha256(path: Path) -> str:
|
||||||
for chunk in iter(lambda: source.read(1024**2), b""):
|
for chunk in iter(lambda: source.read(1024**2), b""):
|
||||||
digest.update(chunk)
|
digest.update(chunk)
|
||||||
return digest.hexdigest()
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _indices(value: npt.NDArray[np.int64], count: int, label: str) -> None:
|
||||||
|
if value.ndim != 1 or np.any(value < 0) or np.any(value >= count):
|
||||||
|
raise GroundSegmentationError(f"{label} indices are invalid")
|
||||||
|
if np.unique(value).size != value.size:
|
||||||
|
raise GroundSegmentationError(f"{label} indices are duplicated")
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,11 @@ from k1link.datasets import (
|
||||||
read_semantic_kitti_frame,
|
read_semantic_kitti_frame,
|
||||||
)
|
)
|
||||||
from k1link.datasets.goose_admission import admit_goose_validation
|
from k1link.datasets.goose_admission import admit_goose_validation
|
||||||
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
|
from k1link.datasets.goose_benchmark import (
|
||||||
|
benchmark_goose_current_ground,
|
||||||
|
benchmark_goose_patchwork_ground,
|
||||||
|
)
|
||||||
|
from k1link.ground_segmentation import GroundSegmentation
|
||||||
from k1link.web.lidar_api import build_lidar_router
|
from k1link.web.lidar_api import build_lidar_router
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -354,12 +358,49 @@ def test_goose_current_ground_is_scored_against_independent_labels(
|
||||||
assert report["ground_truth"]["evaluated_points"] == 4
|
assert report["ground_truth"]["evaluated_points"] == 4
|
||||||
assert 0 <= report["metrics"]["ground_iou"] <= 1
|
assert 0 <= report["metrics"]["ground_iou"] <= 1
|
||||||
assert report["decision"]["promoted"] is False
|
assert report["decision"]["promoted"] is False
|
||||||
output = (
|
output = install / "benchmarks" / f"current-ground-{report['identity_sha256']}" / "preview.json"
|
||||||
install
|
preview = json.loads(output.read_text(encoding="utf-8"))
|
||||||
/ "benchmarks"
|
|
||||||
/ f"current-ground-{report['identity_sha256']}"
|
|
||||||
/ "preview.json"
|
|
||||||
)
|
|
||||||
preview = read_dataset_ground_preview(output)
|
|
||||||
assert preview["point_count"] == 4
|
assert preview["point_count"] == 4
|
||||||
assert len(preview["disagreement"]) == 4
|
assert len(preview["disagreement"]) == 4
|
||||||
|
|
||||||
|
class FakePatchwork:
|
||||||
|
identity = {
|
||||||
|
"provider_id": "patchworkpp/v1.4.1",
|
||||||
|
"source_url": "https://github.com/url-kaist/patchwork-plusplus",
|
||||||
|
"source_tag": "v1.4.1",
|
||||||
|
"source_commit": "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||||
|
"binding_version": "test",
|
||||||
|
"binary_sha256": "c" * 64,
|
||||||
|
"platform": "linux",
|
||||||
|
"machine": "x86_64",
|
||||||
|
"ground_truth": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
def segment(self, xyzi: np.ndarray) -> GroundSegmentation:
|
||||||
|
return GroundSegmentation(
|
||||||
|
ground_mask=np.asarray([True, True, False, False]),
|
||||||
|
assigned_mask=np.ones(xyzi.shape[0], dtype=np.bool_),
|
||||||
|
latency_ms=0.2,
|
||||||
|
)
|
||||||
|
|
||||||
|
ab_report = benchmark_goose_patchwork_ground(
|
||||||
|
root,
|
||||||
|
patchwork=FakePatchwork(),
|
||||||
|
preview_points=4,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert ab_report["scope"]["patchworkpp_input_admitted"] is True
|
||||||
|
assert ab_report["scope"]["normalized_scan_produced"] is False
|
||||||
|
assert ab_report["decision"]["promoted"] is False
|
||||||
|
ab_output = (
|
||||||
|
install / "benchmarks" / f"ground-ab-{ab_report['identity_sha256']}" / "preview.json"
|
||||||
|
)
|
||||||
|
ab_preview = read_dataset_ground_preview(ab_output)
|
||||||
|
assert ab_preview["schema_version"] == "missioncore.dataset-ground-comparison-preview/v2"
|
||||||
|
assert ab_preview["input_profile"]["height"]["sensor_above_ground_m"] == 2.24
|
||||||
|
assert len(ab_preview["patchwork_disagreement"]) == 4
|
||||||
|
|
||||||
|
ab_preview["input_profile"]["height"]["sensor_above_ground_m"] = 2.25
|
||||||
|
ab_output.write_text(json.dumps(ab_preview), encoding="utf-8")
|
||||||
|
with pytest.raises(DatasetAdmissionError, match="input profile"):
|
||||||
|
read_dataset_ground_preview(ab_output)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue