feat(lidar): add ground segmentation diagnostic benchmark
This commit is contained in:
DCCONSTRUCTIONS
parent
e4fdd9fa20
commit
75a3e669d9
|
|
@ -59,6 +59,62 @@ export interface LidarReplayDetail {
|
||||||
stages: LidarStageReadiness[];
|
stages: LidarStageReadiness[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LidarGroundProviderSummary {
|
||||||
|
providerId: string;
|
||||||
|
sourceCommit: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LidarGroundBenchmark {
|
||||||
|
benchmarkId: string;
|
||||||
|
replayPackId: string;
|
||||||
|
sessionId: string;
|
||||||
|
status: "diagnostic-only";
|
||||||
|
frames: number;
|
||||||
|
points: number;
|
||||||
|
inputDomain: {
|
||||||
|
accepted: false;
|
||||||
|
representation: string;
|
||||||
|
physicalSensorHeightKnown: boolean;
|
||||||
|
sensorScanGeometryKnown: boolean;
|
||||||
|
reason: string;
|
||||||
|
};
|
||||||
|
labels: {
|
||||||
|
status: "missing-independent-review";
|
||||||
|
metricsAvailable: false;
|
||||||
|
};
|
||||||
|
current: {
|
||||||
|
provider: LidarGroundProviderSummary;
|
||||||
|
groundFraction: LidarDistribution;
|
||||||
|
assignedFraction: LidarDistribution;
|
||||||
|
latencyMs: LidarDistribution;
|
||||||
|
};
|
||||||
|
candidate: {
|
||||||
|
provider: LidarGroundProviderSummary;
|
||||||
|
groundFraction: LidarDistribution;
|
||||||
|
assignedFraction: LidarDistribution;
|
||||||
|
latencyMs: LidarDistribution;
|
||||||
|
};
|
||||||
|
comparison: {
|
||||||
|
algorithmGroundIou: LidarDistribution;
|
||||||
|
groundDisagreementFraction: LidarDistribution;
|
||||||
|
isAccuracyMetric: false;
|
||||||
|
};
|
||||||
|
decision: {
|
||||||
|
status: "do-not-promote-on-current-vendor-map";
|
||||||
|
productionPromotion: false;
|
||||||
|
reasons: string[];
|
||||||
|
nextGate: string;
|
||||||
|
};
|
||||||
|
createdAtUtc: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LidarGroundBenchmarkCatalog {
|
||||||
|
configured: boolean;
|
||||||
|
validTotal: number;
|
||||||
|
invalidTotal: number;
|
||||||
|
items: LidarGroundBenchmark[];
|
||||||
|
}
|
||||||
|
|
||||||
export class LidarReplayContractError extends Error {}
|
export class LidarReplayContractError extends Error {}
|
||||||
|
|
||||||
export class LidarReplayApiError extends Error {
|
export class LidarReplayApiError extends Error {
|
||||||
|
|
@ -73,8 +129,10 @@ type LidarFetch = (
|
||||||
) => Promise<Response>;
|
) => Promise<Response>;
|
||||||
|
|
||||||
const SAFE_PACK_ID = /^lidar-replay-pack-[a-f0-9]{64}$/;
|
const SAFE_PACK_ID = /^lidar-replay-pack-[a-f0-9]{64}$/;
|
||||||
|
const SAFE_GROUND_BENCHMARK_ID = /^ground-benchmark-[a-f0-9]{64}$/;
|
||||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$/;
|
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$/;
|
||||||
const SHA256 = /^[a-f0-9]{64}$/;
|
const SHA256 = /^[a-f0-9]{64}$/;
|
||||||
|
const GIT_SHA1 = /^[a-f0-9]{40}$/;
|
||||||
|
|
||||||
function record(value: unknown, label: string): Record<string, unknown> {
|
function record(value: unknown, label: string): Record<string, unknown> {
|
||||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||||
|
|
@ -258,6 +316,139 @@ export function parseLidarReplayDetail(value: unknown): LidarReplayDetail {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function groundProvider(
|
||||||
|
value: unknown,
|
||||||
|
label: string,
|
||||||
|
): LidarGroundProviderSummary {
|
||||||
|
const source = record(value, label);
|
||||||
|
return {
|
||||||
|
providerId: string(source.provider_id, `${label}.provider_id`, SAFE_ID),
|
||||||
|
sourceCommit:
|
||||||
|
source.source_commit === undefined || source.source_commit === null
|
||||||
|
? null
|
||||||
|
: string(source.source_commit, `${label}.source_commit`, GIT_SHA1),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function groundBranch(
|
||||||
|
value: unknown,
|
||||||
|
label: string,
|
||||||
|
): LidarGroundBenchmark["current"] {
|
||||||
|
const source = record(value, label);
|
||||||
|
return {
|
||||||
|
provider: groundProvider(source.provider, `${label}.provider`),
|
||||||
|
groundFraction: distribution(
|
||||||
|
source.ground_fraction,
|
||||||
|
`${label}.ground_fraction`,
|
||||||
|
),
|
||||||
|
assignedFraction: distribution(
|
||||||
|
source.assigned_fraction,
|
||||||
|
`${label}.assigned_fraction`,
|
||||||
|
),
|
||||||
|
latencyMs: distribution(source.latency_ms, `${label}.latency_ms`),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function groundBenchmark(value: unknown): LidarGroundBenchmark {
|
||||||
|
const source = record(value, "LiDAR ground benchmark");
|
||||||
|
if (source.status !== "diagnostic-only") {
|
||||||
|
throw new LidarReplayContractError("Ground benchmark status несовместим");
|
||||||
|
}
|
||||||
|
const inputDomain = record(source.input_domain, "input_domain");
|
||||||
|
const labels = record(source.labels, "labels");
|
||||||
|
const comparison = record(source.comparison, "comparison");
|
||||||
|
const decision = record(source.decision, "decision");
|
||||||
|
if (
|
||||||
|
inputDomain.accepted !== false
|
||||||
|
|| labels.status !== "missing-independent-review"
|
||||||
|
|| labels.metrics_available !== false
|
||||||
|
|| comparison.is_accuracy_metric !== false
|
||||||
|
|| decision.status !== "do-not-promote-on-current-vendor-map"
|
||||||
|
|| decision.production_promotion !== false
|
||||||
|
) {
|
||||||
|
throw new LidarReplayContractError(
|
||||||
|
"Ground benchmark завышает readiness или accuracy",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
benchmarkId: string(
|
||||||
|
source.benchmark_id,
|
||||||
|
"benchmark_id",
|
||||||
|
SAFE_GROUND_BENCHMARK_ID,
|
||||||
|
),
|
||||||
|
replayPackId: string(source.replay_pack_id, "replay_pack_id", SAFE_PACK_ID),
|
||||||
|
sessionId: string(source.session_id, "session_id", SAFE_ID),
|
||||||
|
status: "diagnostic-only",
|
||||||
|
frames: integer(source.frames, "frames"),
|
||||||
|
points: integer(source.points, "points"),
|
||||||
|
inputDomain: {
|
||||||
|
accepted: false,
|
||||||
|
representation: string(
|
||||||
|
inputDomain.representation,
|
||||||
|
"input_domain.representation",
|
||||||
|
SAFE_ID,
|
||||||
|
),
|
||||||
|
physicalSensorHeightKnown: boolean(
|
||||||
|
inputDomain.physical_sensor_height_known,
|
||||||
|
"input_domain.physical_sensor_height_known",
|
||||||
|
),
|
||||||
|
sensorScanGeometryKnown: boolean(
|
||||||
|
inputDomain.sensor_scan_geometry_known,
|
||||||
|
"input_domain.sensor_scan_geometry_known",
|
||||||
|
),
|
||||||
|
reason: string(inputDomain.reason, "input_domain.reason"),
|
||||||
|
},
|
||||||
|
labels: {
|
||||||
|
status: "missing-independent-review",
|
||||||
|
metricsAvailable: false,
|
||||||
|
},
|
||||||
|
current: groundBranch(source.current, "current"),
|
||||||
|
candidate: groundBranch(source.candidate, "candidate"),
|
||||||
|
comparison: {
|
||||||
|
algorithmGroundIou: distribution(
|
||||||
|
comparison.algorithm_to_algorithm_ground_iou,
|
||||||
|
"comparison.algorithm_ground_iou",
|
||||||
|
),
|
||||||
|
groundDisagreementFraction: distribution(
|
||||||
|
comparison.ground_disagreement_fraction,
|
||||||
|
"comparison.ground_disagreement_fraction",
|
||||||
|
),
|
||||||
|
isAccuracyMetric: false,
|
||||||
|
},
|
||||||
|
decision: {
|
||||||
|
status: "do-not-promote-on-current-vendor-map",
|
||||||
|
productionPromotion: false,
|
||||||
|
reasons: array(decision.reasons, "decision.reasons").map((reason) =>
|
||||||
|
string(reason, "decision.reason")
|
||||||
|
),
|
||||||
|
nextGate: string(decision.next_gate, "decision.next_gate"),
|
||||||
|
},
|
||||||
|
createdAtUtc:
|
||||||
|
source.created_at_utc === null || source.created_at_utc === undefined
|
||||||
|
? null
|
||||||
|
: string(source.created_at_utc, "created_at_utc"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseLidarGroundBenchmarkCatalog(
|
||||||
|
value: unknown,
|
||||||
|
): LidarGroundBenchmarkCatalog {
|
||||||
|
const source = record(value, "LiDAR ground catalog");
|
||||||
|
if (
|
||||||
|
source.schema_version
|
||||||
|
!== "missioncore.lidar-ground-benchmark-catalog/v1"
|
||||||
|
|| source.access !== "read-only"
|
||||||
|
) {
|
||||||
|
throw new LidarReplayContractError("LiDAR ground catalog несовместим");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
configured: boolean(source.configured, "configured"),
|
||||||
|
validTotal: integer(source.valid_total, "valid_total"),
|
||||||
|
invalidTotal: integer(source.invalid_total, "invalid_total"),
|
||||||
|
items: array(source.items, "items").map(groundBenchmark),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function responseJson(
|
async function responseJson(
|
||||||
response: Response,
|
response: Response,
|
||||||
fallback: string,
|
fallback: string,
|
||||||
|
|
@ -309,3 +500,24 @@ export async function fetchLidarReplayDetail(
|
||||||
await responseJson(response, "Не удалось получить LiDAR quality report."),
|
await responseJson(response, "Не удалось получить LiDAR quality report."),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function fetchLidarGroundBenchmarks(
|
||||||
|
packId: string,
|
||||||
|
options: { signal?: AbortSignal; fetcher?: LidarFetch } = {},
|
||||||
|
): Promise<LidarGroundBenchmarkCatalog> {
|
||||||
|
if (!SAFE_PACK_ID.test(packId)) {
|
||||||
|
throw new LidarReplayContractError("Некорректный LiDAR pack id");
|
||||||
|
}
|
||||||
|
const fetcher = options.fetcher ?? fetch;
|
||||||
|
const response = await fetcher(
|
||||||
|
`/api/v1/lidar/ground-benchmarks?pack_id=${encodeURIComponent(packId)}&limit=20`,
|
||||||
|
{
|
||||||
|
method: "GET",
|
||||||
|
headers: { Accept: "application/json" },
|
||||||
|
signal: options.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return parseLidarGroundBenchmarkCatalog(
|
||||||
|
await responseJson(response, "Не удалось получить LiDAR ground benchmark."),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,10 @@
|
||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics {
|
||||||
|
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||||
|
}
|
||||||
|
|
||||||
.pipeline-strip {
|
.pipeline-strip {
|
||||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
}
|
}
|
||||||
|
|
@ -162,6 +166,11 @@
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics,
|
||||||
|
.lidar-ground-gates {
|
||||||
|
grid-template-columns: 1fr;
|
||||||
|
}
|
||||||
|
|
||||||
.polygon-run-identity dl {
|
.polygon-run-identity dl {
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -954,6 +954,76 @@
|
||||||
margin-top: 0.8rem;
|
margin-top: 0.8rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.lidar-ground-benchmark {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||||
|
gap: 0.65rem;
|
||||||
|
margin-top: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics > div {
|
||||||
|
display: grid;
|
||||||
|
gap: 0.3rem;
|
||||||
|
border: 1px solid var(--station-hairline);
|
||||||
|
border-radius: 0.8rem;
|
||||||
|
background: rgb(255 255 255 / 0.025);
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics span,
|
||||||
|
.lidar-ground-metrics small,
|
||||||
|
.lidar-ground-gates p,
|
||||||
|
.lidar-ground-footer,
|
||||||
|
.lidar-ground-empty {
|
||||||
|
color: var(--nodedc-text-muted);
|
||||||
|
font-size: 0.62rem;
|
||||||
|
line-height: 1.45;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-metrics strong {
|
||||||
|
color: var(--nodedc-text-primary);
|
||||||
|
font-size: 1.15rem;
|
||||||
|
font-weight: 650;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-gates {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||||
|
gap: 0.65rem;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-gates > div {
|
||||||
|
display: grid;
|
||||||
|
align-content: start;
|
||||||
|
gap: 0.55rem;
|
||||||
|
border-left: 1px solid var(--station-hairline);
|
||||||
|
padding-left: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-gates p,
|
||||||
|
.lidar-ground-empty {
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-footer {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin-top: 0.85rem;
|
||||||
|
border-top: 1px solid var(--station-hairline);
|
||||||
|
padding-top: 0.7rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.lidar-ground-empty {
|
||||||
|
margin-top: 1rem;
|
||||||
|
}
|
||||||
|
|
||||||
.overview-grid {
|
.overview-grid {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(0, 1.55fr) minmax(19rem, 0.75fr);
|
grid-template-columns: minmax(0, 1.55fr) minmax(19rem, 0.75fr);
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,10 @@ import {
|
||||||
} from "@nodedc/ui-react";
|
} from "@nodedc/ui-react";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
fetchLidarGroundBenchmarks,
|
||||||
fetchLidarReplayCatalog,
|
fetchLidarReplayCatalog,
|
||||||
fetchLidarReplayDetail,
|
fetchLidarReplayDetail,
|
||||||
|
type LidarGroundBenchmark,
|
||||||
type LidarReplayCatalog,
|
type LidarReplayCatalog,
|
||||||
type LidarReplayDetail,
|
type LidarReplayDetail,
|
||||||
type LidarStageReadiness,
|
type LidarStageReadiness,
|
||||||
|
|
@ -53,6 +55,8 @@ export function LidarQualityWorkspace({
|
||||||
}) {
|
}) {
|
||||||
const [catalog, setCatalog] = useState<LidarReplayCatalog | null>(null);
|
const [catalog, setCatalog] = useState<LidarReplayCatalog | null>(null);
|
||||||
const [detail, setDetail] = useState<LidarReplayDetail | null>(null);
|
const [detail, setDetail] = useState<LidarReplayDetail | null>(null);
|
||||||
|
const [groundBenchmark, setGroundBenchmark] =
|
||||||
|
useState<LidarGroundBenchmark | null>(null);
|
||||||
const [selectedPackId, setSelectedPackId] = useState<string | null>(null);
|
const [selectedPackId, setSelectedPackId] = useState<string | null>(null);
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
@ -72,17 +76,25 @@ export function LidarQualityWorkspace({
|
||||||
const target = selectedPackId ?? nextCatalog.items[0]?.packId ?? null;
|
const target = selectedPackId ?? nextCatalog.items[0]?.packId ?? null;
|
||||||
if (!target) {
|
if (!target) {
|
||||||
setDetail(null);
|
setDetail(null);
|
||||||
|
setGroundBenchmark(null);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const nextDetail = await fetchLidarReplayDetail(target, {
|
const [nextDetail, groundCatalog] = await Promise.all([
|
||||||
signal: controller.signal,
|
fetchLidarReplayDetail(target, {
|
||||||
});
|
signal: controller.signal,
|
||||||
|
}),
|
||||||
|
fetchLidarGroundBenchmarks(target, {
|
||||||
|
signal: controller.signal,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
if (controller.signal.aborted) return;
|
if (controller.signal.aborted) return;
|
||||||
setSelectedPackId(target);
|
setSelectedPackId(target);
|
||||||
setDetail(nextDetail);
|
setDetail(nextDetail);
|
||||||
|
setGroundBenchmark(groundCatalog.items[0] ?? null);
|
||||||
} catch (loadError) {
|
} catch (loadError) {
|
||||||
if (controller.signal.aborted) return;
|
if (controller.signal.aborted) return;
|
||||||
setDetail(null);
|
setDetail(null);
|
||||||
|
setGroundBenchmark(null);
|
||||||
setError(errorMessage(loadError));
|
setError(errorMessage(loadError));
|
||||||
} finally {
|
} finally {
|
||||||
if (!controller.signal.aborted) setLoading(false);
|
if (!controller.signal.aborted) setLoading(false);
|
||||||
|
|
@ -226,6 +238,118 @@ export function LidarQualityWorkspace({
|
||||||
</GlassSurface>
|
</GlassSurface>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<GlassSurface className="lidar-ground-benchmark" padding="lg">
|
||||||
|
<header className="panel-heading">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">GROUND A/B · L2</span>
|
||||||
|
<h2>Текущий heuristic против Patchwork++</h2>
|
||||||
|
</div>
|
||||||
|
<StatusBadge tone={groundBenchmark ? "warning" : "neutral"}>
|
||||||
|
{groundBenchmark ? "Diagnostic only" : "Нет результата"}
|
||||||
|
</StatusBadge>
|
||||||
|
</header>
|
||||||
|
{groundBenchmark ? (
|
||||||
|
<>
|
||||||
|
<section
|
||||||
|
className="lidar-ground-metrics"
|
||||||
|
aria-label="Ground segmentation A/B"
|
||||||
|
>
|
||||||
|
<div>
|
||||||
|
<span>Текущий ground p50</span>
|
||||||
|
<strong>
|
||||||
|
{formatFraction(
|
||||||
|
groundBenchmark.current.groundFraction.p50,
|
||||||
|
)}
|
||||||
|
</strong>
|
||||||
|
<small>
|
||||||
|
{formatNumber(
|
||||||
|
groundBenchmark.current.latencyMs.p95,
|
||||||
|
2,
|
||||||
|
)}{" "}
|
||||||
|
мс p95
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span>Patchwork++ ground p50</span>
|
||||||
|
<strong>
|
||||||
|
{formatFraction(
|
||||||
|
groundBenchmark.candidate.groundFraction.p50,
|
||||||
|
)}
|
||||||
|
</strong>
|
||||||
|
<small>
|
||||||
|
{formatNumber(
|
||||||
|
groundBenchmark.candidate.latencyMs.p95,
|
||||||
|
2,
|
||||||
|
)}{" "}
|
||||||
|
мс p95
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span>Algorithm IoU p50</span>
|
||||||
|
<strong>
|
||||||
|
{formatFraction(
|
||||||
|
groundBenchmark.comparison.algorithmGroundIou.p50,
|
||||||
|
)}
|
||||||
|
</strong>
|
||||||
|
<small>Не является accuracy</small>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<span>Disagreement p50</span>
|
||||||
|
<strong>
|
||||||
|
{formatFraction(
|
||||||
|
groundBenchmark.comparison
|
||||||
|
.groundDisagreementFraction.p50,
|
||||||
|
)}
|
||||||
|
</strong>
|
||||||
|
<small>
|
||||||
|
{groundBenchmark.frames.toLocaleString("ru-RU")} кадров
|
||||||
|
</small>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
<div className="lidar-ground-gates">
|
||||||
|
<div>
|
||||||
|
<StatusBadge tone="danger">
|
||||||
|
Входной контракт не принят
|
||||||
|
</StatusBadge>
|
||||||
|
<p>
|
||||||
|
Patchwork++ ожидает sensor-centric scan и физическую высоту
|
||||||
|
сенсора; текущий point feed является vendor-mapped increment.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<StatusBadge tone="warning">Разметка не принята</StatusBadge>
|
||||||
|
<p>
|
||||||
|
IoU, curb recall, low-obstacle recall и reflection-noise
|
||||||
|
rejection появятся только после независимого human review.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<StatusBadge tone="danger">Не продвигать</StatusBadge>
|
||||||
|
<p>
|
||||||
|
Следующий gate: human-reviewed annotation subset или
|
||||||
|
принятый raw sensor scan с физической высотой сенсора.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<footer className="lidar-ground-footer">
|
||||||
|
<span>
|
||||||
|
{groundBenchmark.current.provider.providerId}
|
||||||
|
</span>
|
||||||
|
<span>
|
||||||
|
{groundBenchmark.candidate.provider.providerId}
|
||||||
|
{groundBenchmark.candidate.provider.sourceCommit
|
||||||
|
? ` · ${groundBenchmark.candidate.provider.sourceCommit.slice(0, 12)}`
|
||||||
|
: ""}
|
||||||
|
</span>
|
||||||
|
</footer>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<p className="lidar-ground-empty">
|
||||||
|
Для выбранного replay pack ещё нет проверенного ground benchmark.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</GlassSurface>
|
||||||
|
|
||||||
<GlassSurface className="lidar-pack-catalog" padding="lg">
|
<GlassSurface className="lidar-pack-catalog" padding="lg">
|
||||||
<header className="panel-heading">
|
<header className="panel-heading">
|
||||||
<div>
|
<div>
|
||||||
|
|
|
||||||
|
|
@ -6,8 +6,10 @@ import { createServer } from "vite";
|
||||||
let server;
|
let server;
|
||||||
let parseLidarReplayCatalog;
|
let parseLidarReplayCatalog;
|
||||||
let parseLidarReplayDetail;
|
let parseLidarReplayDetail;
|
||||||
|
let parseLidarGroundBenchmarkCatalog;
|
||||||
let fetchLidarReplayCatalog;
|
let fetchLidarReplayCatalog;
|
||||||
let fetchLidarReplayDetail;
|
let fetchLidarReplayDetail;
|
||||||
|
let fetchLidarGroundBenchmarks;
|
||||||
let LidarReplayContractError;
|
let LidarReplayContractError;
|
||||||
let workspaceById;
|
let workspaceById;
|
||||||
|
|
||||||
|
|
@ -20,8 +22,10 @@ before(async () => {
|
||||||
({
|
({
|
||||||
parseLidarReplayCatalog,
|
parseLidarReplayCatalog,
|
||||||
parseLidarReplayDetail,
|
parseLidarReplayDetail,
|
||||||
|
parseLidarGroundBenchmarkCatalog,
|
||||||
fetchLidarReplayCatalog,
|
fetchLidarReplayCatalog,
|
||||||
fetchLidarReplayDetail,
|
fetchLidarReplayDetail,
|
||||||
|
fetchLidarGroundBenchmarks,
|
||||||
LidarReplayContractError,
|
LidarReplayContractError,
|
||||||
} = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts"));
|
} = await server.ssrLoadModule("/src/core/lidar/replayQuality.ts"));
|
||||||
({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts"));
|
({ workspaceById } = await server.ssrLoadModule("/src/productModel.ts"));
|
||||||
|
|
@ -130,6 +134,90 @@ function detail() {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function groundCatalog(overrides = {}) {
|
||||||
|
const provider = (providerId, sourceCommit = undefined) => ({
|
||||||
|
provider_id: providerId,
|
||||||
|
source_commit: sourceCommit,
|
||||||
|
});
|
||||||
|
const branch = (providerValue, groundP50, latencyP95) => ({
|
||||||
|
provider: providerValue,
|
||||||
|
ground_fraction: {
|
||||||
|
...distribution(),
|
||||||
|
p50: groundP50,
|
||||||
|
},
|
||||||
|
assigned_fraction: {
|
||||||
|
...distribution(),
|
||||||
|
minimum: 1,
|
||||||
|
mean: 1,
|
||||||
|
p50: 1,
|
||||||
|
p95: 1,
|
||||||
|
maximum: 1,
|
||||||
|
},
|
||||||
|
latency_ms: {
|
||||||
|
...distribution(),
|
||||||
|
p95: latencyP95,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
schema_version: "missioncore.lidar-ground-benchmark-catalog/v1",
|
||||||
|
configured: true,
|
||||||
|
valid_total: 1,
|
||||||
|
invalid_total: 0,
|
||||||
|
access: "read-only",
|
||||||
|
items: [{
|
||||||
|
benchmark_id: `ground-benchmark-${"c".repeat(64)}`,
|
||||||
|
replay_pack_id: packId,
|
||||||
|
session_id: "20260719T220917Z_viewer_live",
|
||||||
|
status: "diagnostic-only",
|
||||||
|
frames: 66,
|
||||||
|
points: 226963,
|
||||||
|
input_domain: {
|
||||||
|
accepted: false,
|
||||||
|
representation: "vendor-map-increment",
|
||||||
|
physical_sensor_height_known: false,
|
||||||
|
sensor_scan_geometry_known: false,
|
||||||
|
reason: "Patchwork++ expects sensor-centric scans.",
|
||||||
|
},
|
||||||
|
labels: {
|
||||||
|
status: "missing-independent-review",
|
||||||
|
metrics_available: false,
|
||||||
|
},
|
||||||
|
current: branch(
|
||||||
|
provider("missioncore-local-percentile-ground/v1"),
|
||||||
|
0.183,
|
||||||
|
10.7,
|
||||||
|
),
|
||||||
|
candidate: branch(
|
||||||
|
provider(
|
||||||
|
"patchworkpp/v1.4.1",
|
||||||
|
"3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
||||||
|
),
|
||||||
|
0.005,
|
||||||
|
0.29,
|
||||||
|
),
|
||||||
|
comparison: {
|
||||||
|
algorithm_to_algorithm_ground_iou: {
|
||||||
|
...distribution(),
|
||||||
|
p50: 0.029,
|
||||||
|
},
|
||||||
|
ground_disagreement_fraction: {
|
||||||
|
...distribution(),
|
||||||
|
p50: 0.179,
|
||||||
|
},
|
||||||
|
is_accuracy_metric: false,
|
||||||
|
},
|
||||||
|
decision: {
|
||||||
|
status: "do-not-promote-on-current-vendor-map",
|
||||||
|
production_promotion: false,
|
||||||
|
reasons: ["candidate input domain is not accepted"],
|
||||||
|
next_gate: "human-reviewed annotation subset",
|
||||||
|
},
|
||||||
|
created_at_utc: "2026-07-25T01:00:00Z",
|
||||||
|
}],
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function jsonResponse(payload, status = 200) {
|
function jsonResponse(payload, status = 200) {
|
||||||
return new Response(JSON.stringify(payload), {
|
return new Response(JSON.stringify(payload), {
|
||||||
status,
|
status,
|
||||||
|
|
@ -157,22 +245,48 @@ test("LiDAR contract refuses replay that did not pass equivalence", () => {
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("ground benchmark stays diagnostic until input and labels are accepted", () => {
|
||||||
|
const parsed = parseLidarGroundBenchmarkCatalog(groundCatalog());
|
||||||
|
|
||||||
|
assert.equal(parsed.items[0].status, "diagnostic-only");
|
||||||
|
assert.equal(parsed.items[0].inputDomain.accepted, false);
|
||||||
|
assert.equal(parsed.items[0].labels.metricsAvailable, false);
|
||||||
|
assert.equal(parsed.items[0].candidate.groundFraction.p50, 0.005);
|
||||||
|
assert.equal(parsed.items[0].decision.productionPromotion, false);
|
||||||
|
|
||||||
|
const promoted = groundCatalog();
|
||||||
|
promoted.items[0].decision.production_promotion = true;
|
||||||
|
assert.throws(
|
||||||
|
() => parseLidarGroundBenchmarkCatalog(promoted),
|
||||||
|
LidarReplayContractError,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => {
|
test("LiDAR fetchers use read-only endpoints and workspace is registered", async () => {
|
||||||
const calls = [];
|
const calls = [];
|
||||||
const fetcher = async (input, init) => {
|
const fetcher = async (input, init) => {
|
||||||
calls.push({ input: String(input), method: init?.method });
|
calls.push({ input: String(input), method: init?.method });
|
||||||
|
if (String(input).includes("ground-benchmarks")) {
|
||||||
|
return jsonResponse(groundCatalog());
|
||||||
|
}
|
||||||
return String(input).includes(packId)
|
return String(input).includes(packId)
|
||||||
? jsonResponse(detail())
|
? jsonResponse(detail())
|
||||||
: jsonResponse(catalog());
|
: jsonResponse(catalog());
|
||||||
};
|
};
|
||||||
const parsedCatalog = await fetchLidarReplayCatalog({ fetcher });
|
const parsedCatalog = await fetchLidarReplayCatalog({ fetcher });
|
||||||
const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher });
|
const parsedDetail = await fetchLidarReplayDetail(packId, { fetcher });
|
||||||
|
const ground = await fetchLidarGroundBenchmarks(packId, { fetcher });
|
||||||
|
|
||||||
assert.equal(parsedCatalog.validTotal, 1);
|
assert.equal(parsedCatalog.validTotal, 1);
|
||||||
assert.equal(parsedDetail.pack.packId, packId);
|
assert.equal(parsedDetail.pack.packId, packId);
|
||||||
|
assert.equal(ground.validTotal, 1);
|
||||||
assert.deepEqual(calls, [
|
assert.deepEqual(calls, [
|
||||||
{ input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" },
|
{ input: "/api/v1/lidar/replay-packs?limit=50", method: "GET" },
|
||||||
{ input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" },
|
{ input: `/api/v1/lidar/replay-packs/${packId}`, method: "GET" },
|
||||||
|
{
|
||||||
|
input: `/api/v1/lidar/ground-benchmarks?pack_id=${packId}&limit=20`,
|
||||||
|
method: "GET",
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
assert.equal(workspaceById("lidar-quality").root, "data");
|
assert.equal(workspaceById("lidar-quality").root, "data");
|
||||||
assert.equal(workspaceById("lidar-quality").kind, "lidar-quality");
|
assert.equal(workspaceById("lidar-quality").kind, "lidar-quality");
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,16 @@ equivalence report. The React surface reads those reports through the read-only
|
||||||
`/api/v1/lidar/replay-packs` boundary; CUDA/TensorRT and ROS do not move into the
|
`/api/v1/lidar/replay-packs` boundary; CUDA/TensorRT and ROS do not move into the
|
||||||
browser.
|
browser.
|
||||||
|
|
||||||
|
`missioncore.lidar-ground-benchmark/v1` extends the same boundary for L2. It
|
||||||
|
keeps point-aligned ground/assigned masks for the current local-percentile
|
||||||
|
proposal and a candidate provider, exact provider/source/binary identities,
|
||||||
|
host latency and algorithm disagreement. The first official Patchwork++ v1.4.1
|
||||||
|
run is diagnostic-only: current K1 evidence is a vendor-map increment, not the
|
||||||
|
sensor-centric scan and physical-height contract Patchwork++ expects. The
|
||||||
|
read-only `/api/v1/lidar/ground-benchmarks` surface therefore publishes
|
||||||
|
`production_promotion=false` until an independent annotation generation or an
|
||||||
|
admitted raw scan closes the input gate.
|
||||||
|
|
||||||
## Boundary
|
## Boundary
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
# LiDAR worker: product value, evidence boundary and implementation roadmap
|
# LiDAR worker: product value, evidence boundary and implementation roadmap
|
||||||
|
|
||||||
Date: 2026-07-25
|
Date: 2026-07-25
|
||||||
Status: accepted architecture plan; L0 and L1 implemented
|
Status: accepted architecture plan; L0/L1 implemented; L2 diagnostic A/B complete,
|
||||||
|
independent labels pending
|
||||||
Scope: real scanner records, replay and future live shadow processing
|
Scope: real scanner records, replay and future live shadow processing
|
||||||
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
Explicitly out of scope: Unreal U0/U1, Gaussian assets and simulator rendering
|
||||||
|
|
||||||
|
|
@ -111,7 +112,7 @@ and training-domain fit.
|
||||||
|
|
||||||
| Component | Correct use | Decision |
|
| Component | Correct use | Decision |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | First non-neural geometry baseline |
|
| [Patchwork++](https://github.com/url-kaist/patchwork-plusplus) | Fast adaptive ground segmentation, including reflection-noise handling | v1.4.1 benchmarked; do not promote on the current vendor-map feed |
|
||||||
| [Autoware CenterPoint](https://github.com/autowarefoundation/autoware_universe/tree/main/perception/autoware_lidar_centerpoint) | Mature ROS 2/TensorRT 3D detection and multi-frame reference | Second detector baseline after PointPillars |
|
| [Autoware CenterPoint](https://github.com/autowarefoundation/autoware_universe/tree/main/perception/autoware_lidar_centerpoint) | Mature ROS 2/TensorRT 3D detection and multi-frame reference | Second detector baseline after PointPillars |
|
||||||
| [MMDetection3D](https://github.com/open-mmlab/mmdetection3d) | Training/evaluation harness and dataset adapters | Laboratory only |
|
| [MMDetection3D](https://github.com/open-mmlab/mmdetection3d) | Training/evaluation harness and dataset adapters | Laboratory only |
|
||||||
| [OpenPCDet](https://github.com/open-mmlab/OpenPCDet) | Alternative LiDAR detector benchmark/model zoo | Laboratory only; not the production runtime |
|
| [OpenPCDet](https://github.com/open-mmlab/OpenPCDet) | Alternative LiDAR detector benchmark/model zoo | Laboratory only; not the production runtime |
|
||||||
|
|
@ -192,17 +193,43 @@ Accepted real slice:
|
||||||
The long interval tail is evidence to investigate in the scanner/transport
|
The long interval tail is evidence to investigate in the scanner/transport
|
||||||
quality line. It is not repaired or hidden by replay.
|
quality line. It is not repaired or hidden by replay.
|
||||||
|
|
||||||
### L2 — geometric baseline
|
### L2 — geometric baseline — diagnostic A/B complete, accuracy gate open
|
||||||
|
|
||||||
- [ ] Run Patchwork++ over the frozen XYZI slice.
|
- [x] Pin and run official Patchwork++ v1.4.1 at source commit
|
||||||
- [ ] Retain ground and non-ground outputs separately; never delete raw points.
|
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`.
|
||||||
- [ ] Label a small independent ground/obstacle evaluation set in CVAT or an
|
- [x] Compare it against a full-frame, point-aligned extension of the current
|
||||||
equivalent accepted annotation workspace.
|
E19 local-percentile ground proposal.
|
||||||
- [ ] Measure ground IoU, curb/low-obstacle recall, reflection-noise rejection
|
- [x] Retain separate current/candidate ground and assigned masks for every
|
||||||
and CPU/GPU latency.
|
source point; raw replay remains unchanged.
|
||||||
- [ ] Compare against the current heuristic ground proposal branch.
|
- [x] Seal latency, ground-fraction, algorithm-IoU and disagreement
|
||||||
|
distributions in `missioncore.lidar-ground-benchmark/v1`.
|
||||||
|
- [x] Create an immutable eight-frame annotation template in which every point
|
||||||
|
starts as `ignore-unreviewed`; it is explicitly not ground truth.
|
||||||
|
- [ ] Complete independent human review for ground, curb, low obstacle,
|
||||||
|
reflection noise and other non-ground points.
|
||||||
|
- [ ] Measure accepted ground IoU, curb/low-obstacle recall and
|
||||||
|
reflection-noise rejection against that reviewed generation.
|
||||||
|
|
||||||
Exit: an evidence-backed decision to retain or reject Patchwork++.
|
The real diagnostic run is
|
||||||
|
`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`.
|
||||||
|
It covers all 66 frames and 226,963 points. The current local-percentile
|
||||||
|
proposal classified 18.31% ground at p50 with 9.74 ms p95 host latency.
|
||||||
|
Patchwork++ classified only 0.53% ground at p50 with 0.26 ms p95 host latency.
|
||||||
|
Their point-aligned ground IoU was 2.90% p50 and disagreement was 17.92% p50.
|
||||||
|
These last two values compare algorithms; they are not accuracy metrics.
|
||||||
|
|
||||||
|
The result is an evidence-backed **do-not-promote** decision for the current K1
|
||||||
|
feed. Patchwork++ is fast, but its input model assumes a sensor-centric scan
|
||||||
|
and physical sensor height. K1 `lio_pcl` is a vendor-mapped increment, its
|
||||||
|
physical sensor height is not encoded by the best-effort pose, and scan
|
||||||
|
geometry remains unknown. Translating the cloud until Patchwork++ looks
|
||||||
|
plausible would tune against the candidate and invalidate the comparison.
|
||||||
|
|
||||||
|
The independent-label exit remains open. It can be closed by reviewing the
|
||||||
|
content-bound template
|
||||||
|
`ground-annotation-template-12e12eab0756c14f5e06adcaf13189e93188df9bb6d31224cb9b7d566ec15b18`,
|
||||||
|
or by acquiring an admitted raw sensor scan with known physical sensor height
|
||||||
|
and then producing a new benchmark generation.
|
||||||
|
|
||||||
### L3 — LiDAR-native 3D detection
|
### L3 — LiDAR-native 3D detection
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
# ADR 0020: do not promote Patchwork++ on vendor-mapped LiDAR increments
|
||||||
|
|
||||||
|
Date: 2026-07-25
|
||||||
|
Status: accepted and implemented as a diagnostic gate
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
The L2 roadmap selected Patchwork++ as the first non-neural ground segmentation
|
||||||
|
baseline. The official implementation is designed around a sensor-centric
|
||||||
|
LiDAR scan, radial zones and a physical sensor height. Current K1 `lio_pcl`
|
||||||
|
evidence instead contains a vendor-mapped point increment in `map`, an
|
||||||
|
independent best-effort pose and no admitted raw sweep or scan geometry. The
|
||||||
|
pose describes vendor odometry; it does not prove the physical height of the
|
||||||
|
LiDAR above terrain.
|
||||||
|
|
||||||
|
Treating a successful Patchwork++ call as a valid baseline would conflate API
|
||||||
|
compatibility with input-domain compatibility. Choosing a synthetic Z
|
||||||
|
translation until the output looks plausible would use the candidate itself to
|
||||||
|
define the normalization.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
1. Pin the official Patchwork++ v1.4.1 source at commit
|
||||||
|
`3e6903a1d5537a4cc2ace897b0bbb98a92d6014c`.
|
||||||
|
2. Run it only behind the provider-neutral `lidar-ground/v1` diagnostic
|
||||||
|
boundary.
|
||||||
|
3. Bind the exact source commit and compiled binary SHA-256 into every result.
|
||||||
|
4. Preserve point-aligned ground and assigned masks separately from raw replay.
|
||||||
|
5. Compare it with the current local-percentile proposal using
|
||||||
|
algorithm-to-algorithm IoU, disagreement and latency.
|
||||||
|
6. Mark input-domain acceptance, labeled accuracy and production promotion
|
||||||
|
false for current K1 vendor-map evidence.
|
||||||
|
7. Never present algorithm IoU as ground IoU.
|
||||||
|
8. Create an all-ignore, content-bound annotation template; only a separate
|
||||||
|
human-reviewed generation may unlock ground IoU, curb/low-obstacle recall
|
||||||
|
and reflection-noise rejection.
|
||||||
|
9. Keep command, navigation and safety authority false.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Patchwork++ remains a useful candidate for a future raw vehicle LiDAR feed.
|
||||||
|
- Its sub-millisecond host latency on the current slice does not compensate for
|
||||||
|
the unaccepted input domain.
|
||||||
|
- Current K1 work should prioritize independent labels and source evidence, not
|
||||||
|
threshold tuning around a mis-specified sensor model.
|
||||||
|
- The same immutable benchmark/API/React surface can compare a future raw scan,
|
||||||
|
replay or simulation provider without moving C++ processing into React.
|
||||||
|
|
||||||
|
## Real diagnostic evidence
|
||||||
|
|
||||||
|
Benchmark
|
||||||
|
`ground-benchmark-68cfd7a8f1dd4c0006183bb4f63a23f9ff1dd7459317ff0886e995f6c320d984`
|
||||||
|
processed 66 frames and 226,963 points:
|
||||||
|
|
||||||
|
| Measurement | Current local percentile | Patchwork++ |
|
||||||
|
| --- | ---: | ---: |
|
||||||
|
| Ground fraction p50 | 18.31% | 0.53% |
|
||||||
|
| Host latency p95 | 9.74 ms | 0.26 ms |
|
||||||
|
|
||||||
|
Algorithm ground IoU was 2.90% p50 and point disagreement was 17.92% p50.
|
||||||
|
Neither is an accuracy metric. Independent labels are still missing.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- `src/k1link/compute/lidar_ground.py`
|
||||||
|
- `experiments/perception/run_lidar_ground_benchmark.py`
|
||||||
|
- `src/k1link/web/lidar_api.py`
|
||||||
|
- `apps/control-station/src/workspaces/LidarQualityWorkspace.tsx`
|
||||||
|
- `docs/13_LIDAR_WORKER_PRODUCT_AND_ROADMAP.md`
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from k1link.compute import (
|
||||||
|
PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
PATCHWORKPP_SOURCE_TAG,
|
||||||
|
LidarGroundBenchmarkV1,
|
||||||
|
LidarReplayPackV2,
|
||||||
|
PatchworkPPGroundSegmenter,
|
||||||
|
build_lidar_ground_annotation_template,
|
||||||
|
build_lidar_ground_benchmark,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _arguments() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description=(
|
||||||
|
"Run diagnostic-only local-percentile vs Patchwork++ ground A/B "
|
||||||
|
"against one immutable LiDAR replay pack."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
parser.add_argument("replay_pack", type=Path)
|
||||||
|
parser.add_argument("output_root", type=Path)
|
||||||
|
parser.add_argument(
|
||||||
|
"--annotation-root",
|
||||||
|
type=Path,
|
||||||
|
help="Optional root for an all-ignore human-review template.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--annotation-frames",
|
||||||
|
type=int,
|
||||||
|
default=8,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--patchwork-module",
|
||||||
|
default="pypatchworkpp",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--patchwork-source-tag",
|
||||||
|
default=PATCHWORKPP_SOURCE_TAG,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--patchwork-source-commit",
|
||||||
|
default=PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
arguments = _arguments()
|
||||||
|
replay = LidarReplayPackV2(arguments.replay_pack)
|
||||||
|
try:
|
||||||
|
patchwork = PatchworkPPGroundSegmenter.load(
|
||||||
|
module_name=arguments.patchwork_module,
|
||||||
|
source_tag=arguments.patchwork_source_tag,
|
||||||
|
source_commit=arguments.patchwork_source_commit,
|
||||||
|
)
|
||||||
|
output = build_lidar_ground_benchmark(
|
||||||
|
replay,
|
||||||
|
arguments.output_root,
|
||||||
|
patchwork=patchwork,
|
||||||
|
)
|
||||||
|
annotation = (
|
||||||
|
build_lidar_ground_annotation_template(
|
||||||
|
replay,
|
||||||
|
arguments.annotation_root,
|
||||||
|
requested_frames=arguments.annotation_frames,
|
||||||
|
)
|
||||||
|
if arguments.annotation_root is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
replay.close()
|
||||||
|
|
||||||
|
result = LidarGroundBenchmarkV1(output)
|
||||||
|
try:
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"benchmark_id": result.benchmark_id,
|
||||||
|
"status": result.report["status"],
|
||||||
|
"decision": result.report["decision"],
|
||||||
|
"current": result.report["current"],
|
||||||
|
"candidate": result.report["candidate"],
|
||||||
|
"comparison": result.report["comparison"],
|
||||||
|
"annotation_template_id": (
|
||||||
|
annotation.name if annotation is not None else None
|
||||||
|
),
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
result.close()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -69,6 +69,25 @@ from .lidar_contract import (
|
||||||
lidar_readiness_document,
|
lidar_readiness_document,
|
||||||
sensor_frame_xyzi,
|
sensor_frame_xyzi,
|
||||||
)
|
)
|
||||||
|
from .lidar_ground import (
|
||||||
|
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
|
LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA,
|
||||||
|
LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA,
|
||||||
|
LIDAR_GROUND_BENCHMARK_SCHEMA,
|
||||||
|
PATCHWORKPP_SOURCE_COMMIT,
|
||||||
|
PATCHWORKPP_SOURCE_TAG,
|
||||||
|
PATCHWORKPP_SOURCE_URL,
|
||||||
|
GroundBenchmarkProfile,
|
||||||
|
GroundSegmentation,
|
||||||
|
LidarGroundBenchmarkV1,
|
||||||
|
LidarGroundError,
|
||||||
|
LocalPercentileGroundSegmenter,
|
||||||
|
PatchworkPPGroundSegmenter,
|
||||||
|
build_lidar_ground_annotation_template,
|
||||||
|
build_lidar_ground_benchmark,
|
||||||
|
lidar_ground_benchmark_catalog_item,
|
||||||
|
score_ground_labels,
|
||||||
|
)
|
||||||
from .lidar_replay import (
|
from .lidar_replay import (
|
||||||
LIDAR_EQUIVALENCE_REPORT_SCHEMA,
|
LIDAR_EQUIVALENCE_REPORT_SCHEMA,
|
||||||
LIDAR_QUALITY_REPORT_SCHEMA,
|
LIDAR_QUALITY_REPORT_SCHEMA,
|
||||||
|
|
@ -151,7 +170,12 @@ __all__ = [
|
||||||
"EVALUATION_PACK_SCHEMA",
|
"EVALUATION_PACK_SCHEMA",
|
||||||
"EvaluationFrameRequest",
|
"EvaluationFrameRequest",
|
||||||
"EvaluationPackFrame",
|
"EvaluationPackFrame",
|
||||||
|
"GroundBenchmarkProfile",
|
||||||
|
"GroundSegmentation",
|
||||||
"LatestWinsQueue",
|
"LatestWinsQueue",
|
||||||
|
"LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA",
|
||||||
|
"LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA",
|
||||||
|
"LIDAR_GROUND_BENCHMARK_SCHEMA",
|
||||||
"LIDAR_EVIDENCE_PROFILE_SCHEMA",
|
"LIDAR_EVIDENCE_PROFILE_SCHEMA",
|
||||||
"LIDAR_EQUIVALENCE_REPORT_SCHEMA",
|
"LIDAR_EQUIVALENCE_REPORT_SCHEMA",
|
||||||
"LIDAR_QUALITY_REPORT_SCHEMA",
|
"LIDAR_QUALITY_REPORT_SCHEMA",
|
||||||
|
|
@ -169,6 +193,8 @@ __all__ = [
|
||||||
"LidarPoseStatus",
|
"LidarPoseStatus",
|
||||||
"LidarQualityMonitor",
|
"LidarQualityMonitor",
|
||||||
"LidarReadiness",
|
"LidarReadiness",
|
||||||
|
"LidarGroundBenchmarkV1",
|
||||||
|
"LidarGroundError",
|
||||||
"LidarReplayError",
|
"LidarReplayError",
|
||||||
"LidarReplayPackV2",
|
"LidarReplayPackV2",
|
||||||
"LidarReplayPointFrame",
|
"LidarReplayPointFrame",
|
||||||
|
|
@ -190,6 +216,12 @@ __all__ = [
|
||||||
"K1_LAB_LIDAR_PACK_V1_PROFILE",
|
"K1_LAB_LIDAR_PACK_V1_PROFILE",
|
||||||
"K1_LIDAR_PACK_V2_PROFILE",
|
"K1_LIDAR_PACK_V2_PROFILE",
|
||||||
"K1_LIVE_LIDAR_PROFILE",
|
"K1_LIVE_LIDAR_PROFILE",
|
||||||
|
"DEFAULT_GROUND_BENCHMARK_PROFILE",
|
||||||
|
"LocalPercentileGroundSegmenter",
|
||||||
|
"PATCHWORKPP_SOURCE_COMMIT",
|
||||||
|
"PATCHWORKPP_SOURCE_TAG",
|
||||||
|
"PATCHWORKPP_SOURCE_URL",
|
||||||
|
"PatchworkPPGroundSegmenter",
|
||||||
"QueueSnapshot",
|
"QueueSnapshot",
|
||||||
"RecordedCalibratedFusion",
|
"RecordedCalibratedFusion",
|
||||||
"RecordedCalibratedFusionStore",
|
"RecordedCalibratedFusionStore",
|
||||||
|
|
@ -224,6 +256,8 @@ __all__ = [
|
||||||
"prepare_recorded_qualification_slice",
|
"prepare_recorded_qualification_slice",
|
||||||
"assess_lidar_profile",
|
"assess_lidar_profile",
|
||||||
"build_lidar_replay_pack_v2",
|
"build_lidar_replay_pack_v2",
|
||||||
|
"build_lidar_ground_annotation_template",
|
||||||
|
"build_lidar_ground_benchmark",
|
||||||
"DetectionFrame",
|
"DetectionFrame",
|
||||||
"ObjectDetection",
|
"ObjectDetection",
|
||||||
"RecordedPerceptionOverlayError",
|
"RecordedPerceptionOverlayError",
|
||||||
|
|
@ -246,6 +280,8 @@ __all__ = [
|
||||||
"lidar_readiness_document",
|
"lidar_readiness_document",
|
||||||
"lidar_pack_catalog_item",
|
"lidar_pack_catalog_item",
|
||||||
"lidar_pack_detail",
|
"lidar_pack_detail",
|
||||||
|
"lidar_ground_benchmark_catalog_item",
|
||||||
|
"score_ground_labels",
|
||||||
"sensor_frame_xyzi",
|
"sensor_frame_xyzi",
|
||||||
"verify_lidar_replay_equivalence",
|
"verify_lidar_replay_equivalence",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -407,7 +407,14 @@ app.include_router(
|
||||||
/ "compute-experiments"
|
/ "compute-experiments"
|
||||||
/ "lidar-replay-v2"
|
/ "lidar-replay-v2"
|
||||||
/ "packs"
|
/ "packs"
|
||||||
)
|
),
|
||||||
|
ground_root_provider=lambda: (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ ".runtime"
|
||||||
|
/ "compute-experiments"
|
||||||
|
/ "lidar-ground-v1"
|
||||||
|
/ "benchmarks"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,14 +9,21 @@ from typing import Any, Final
|
||||||
from fastapi import APIRouter, HTTPException, Query
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
|
|
||||||
from k1link.compute import (
|
from k1link.compute import (
|
||||||
|
LidarGroundBenchmarkV1,
|
||||||
|
LidarGroundError,
|
||||||
LidarReplayError,
|
LidarReplayError,
|
||||||
LidarReplayPackV2,
|
LidarReplayPackV2,
|
||||||
|
lidar_ground_benchmark_catalog_item,
|
||||||
lidar_pack_catalog_item,
|
lidar_pack_catalog_item,
|
||||||
lidar_pack_detail,
|
lidar_pack_detail,
|
||||||
)
|
)
|
||||||
|
|
||||||
LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1"
|
LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1"
|
||||||
|
LIDAR_GROUND_CATALOG_SCHEMA: Final = (
|
||||||
|
"missioncore.lidar-ground-benchmark-catalog/v1"
|
||||||
|
)
|
||||||
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
|
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
|
||||||
|
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
|
||||||
RootProvider = Callable[[], Path | None]
|
RootProvider = Callable[[], Path | None]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -25,9 +32,15 @@ def configured_lidar_replay_root() -> Path | None:
|
||||||
return Path(value).expanduser().absolute() if value else None
|
return Path(value).expanduser().absolute() if value else None
|
||||||
|
|
||||||
|
|
||||||
|
def configured_lidar_ground_root() -> Path | None:
|
||||||
|
value = os.environ.get("MISSIONCORE_LIDAR_GROUND_ROOT", "").strip()
|
||||||
|
return Path(value).expanduser().absolute() if value else None
|
||||||
|
|
||||||
|
|
||||||
def build_lidar_router(
|
def build_lidar_router(
|
||||||
*,
|
*,
|
||||||
root_provider: RootProvider = configured_lidar_replay_root,
|
root_provider: RootProvider = configured_lidar_replay_root,
|
||||||
|
ground_root_provider: RootProvider = configured_lidar_ground_root,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||||
|
|
||||||
|
|
@ -107,4 +120,95 @@ def build_lidar_router(
|
||||||
detail="LiDAR replay pack не прошёл проверку целостности",
|
detail="LiDAR replay pack не прошёл проверку целостности",
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
@router.get("/ground-benchmarks")
|
||||||
|
def list_lidar_ground_benchmarks(
|
||||||
|
pack_id: str | None = Query(default=None),
|
||||||
|
limit: int = Query(default=20, ge=1, le=100),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if pack_id is not None and _PACK_ID.fullmatch(pack_id) is None:
|
||||||
|
raise HTTPException(status_code=404, detail="LiDAR replay pack не найден")
|
||||||
|
root = ground_root_provider()
|
||||||
|
if root is None or not root.is_dir():
|
||||||
|
return {
|
||||||
|
"schema_version": LIDAR_GROUND_CATALOG_SCHEMA,
|
||||||
|
"configured": root is not None,
|
||||||
|
"items": [],
|
||||||
|
"valid_total": 0,
|
||||||
|
"invalid_total": 0,
|
||||||
|
"access": "read-only",
|
||||||
|
}
|
||||||
|
items: list[dict[str, object]] = []
|
||||||
|
invalid_total = 0
|
||||||
|
candidates = sorted(
|
||||||
|
(
|
||||||
|
candidate
|
||||||
|
for candidate in root.iterdir()
|
||||||
|
if candidate.is_dir()
|
||||||
|
and _BENCHMARK_ID.fullmatch(candidate.name) is not None
|
||||||
|
),
|
||||||
|
key=lambda candidate: candidate.stat().st_mtime_ns,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
for candidate in candidates:
|
||||||
|
try:
|
||||||
|
benchmark = LidarGroundBenchmarkV1(candidate)
|
||||||
|
try:
|
||||||
|
if (
|
||||||
|
pack_id is None
|
||||||
|
or benchmark.identity.get("replay_pack_id") == pack_id
|
||||||
|
):
|
||||||
|
items.append(
|
||||||
|
lidar_ground_benchmark_catalog_item(benchmark)
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
benchmark.close()
|
||||||
|
except (LidarGroundError, OSError):
|
||||||
|
invalid_total += 1
|
||||||
|
return {
|
||||||
|
"schema_version": LIDAR_GROUND_CATALOG_SCHEMA,
|
||||||
|
"configured": True,
|
||||||
|
"items": items[:limit],
|
||||||
|
"valid_total": len(items),
|
||||||
|
"invalid_total": invalid_total,
|
||||||
|
"access": "read-only",
|
||||||
|
}
|
||||||
|
|
||||||
|
@router.get("/ground-benchmarks/{benchmark_id}")
|
||||||
|
def get_lidar_ground_benchmark(benchmark_id: str) -> dict[str, object]:
|
||||||
|
if _BENCHMARK_ID.fullmatch(benchmark_id) is None:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail="LiDAR ground benchmark не найден",
|
||||||
|
)
|
||||||
|
root = ground_root_provider()
|
||||||
|
if root is None or not root.is_dir():
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail="LiDAR ground storage не настроен",
|
||||||
|
)
|
||||||
|
candidate = root / benchmark_id
|
||||||
|
if not candidate.is_dir():
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=404,
|
||||||
|
detail="LiDAR ground benchmark не найден",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
benchmark = LidarGroundBenchmarkV1(candidate)
|
||||||
|
try:
|
||||||
|
return {
|
||||||
|
"schema_version": (
|
||||||
|
"missioncore.lidar-ground-benchmark-detail/v1"
|
||||||
|
),
|
||||||
|
"benchmark": lidar_ground_benchmark_catalog_item(benchmark),
|
||||||
|
"report": benchmark.report,
|
||||||
|
"access": "read-only",
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
benchmark.close()
|
||||||
|
except (LidarGroundError, OSError) as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=409,
|
||||||
|
detail="LiDAR ground benchmark не прошёл проверку целостности",
|
||||||
|
) from exc
|
||||||
|
|
||||||
return router
|
return router
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,306 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
from fastapi import APIRouter
|
||||||
|
from fastapi.routing import APIRoute
|
||||||
|
|
||||||
|
from k1link.compute import (
|
||||||
|
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
|
K1_LIDAR_PACK_V2_PROFILE,
|
||||||
|
GroundSegmentation,
|
||||||
|
LidarGroundBenchmarkV1,
|
||||||
|
LidarGroundError,
|
||||||
|
LidarReplayPointFrame,
|
||||||
|
LidarReplayPoseFrame,
|
||||||
|
LocalPercentileGroundSegmenter,
|
||||||
|
PatchworkPPGroundSegmenter,
|
||||||
|
build_lidar_ground_annotation_template,
|
||||||
|
build_lidar_ground_benchmark,
|
||||||
|
score_ground_labels,
|
||||||
|
)
|
||||||
|
from k1link.web.lidar_api import build_lidar_router
|
||||||
|
|
||||||
|
|
||||||
|
class _Replay:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.pack_id = f"lidar-replay-pack-{'a' * 64}"
|
||||||
|
self.identity = {
|
||||||
|
"logical_content_sha256": "b" * 64,
|
||||||
|
"session_id": "synthetic-ground-session",
|
||||||
|
}
|
||||||
|
self.profile = K1_LIDAR_PACK_V2_PROFILE
|
||||||
|
self._points = (
|
||||||
|
np.asarray(
|
||||||
|
[
|
||||||
|
[-1.0, -1.0, 0.00],
|
||||||
|
[-0.5, -1.0, 0.02],
|
||||||
|
[0.0, -1.0, 0.01],
|
||||||
|
[0.5, -1.0, 0.03],
|
||||||
|
[1.0, -1.0, 0.00],
|
||||||
|
[-1.0, 0.0, 0.01],
|
||||||
|
[-0.5, 0.0, 0.02],
|
||||||
|
[0.0, 0.0, 0.04],
|
||||||
|
[0.5, 0.0, 0.35],
|
||||||
|
[1.0, 0.0, 0.70],
|
||||||
|
],
|
||||||
|
dtype=np.float64,
|
||||||
|
),
|
||||||
|
np.asarray(
|
||||||
|
[
|
||||||
|
[-1.0, -1.0, 0.01],
|
||||||
|
[-0.5, -1.0, 0.01],
|
||||||
|
[0.0, -1.0, 0.02],
|
||||||
|
[0.5, -1.0, 0.02],
|
||||||
|
[1.0, -1.0, 0.01],
|
||||||
|
[-1.0, 0.0, 0.02],
|
||||||
|
[-0.5, 0.0, 0.01],
|
||||||
|
[0.0, 0.0, 0.03],
|
||||||
|
[0.5, 0.0, 0.40],
|
||||||
|
[1.0, 0.0, 0.80],
|
||||||
|
],
|
||||||
|
dtype=np.float64,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.arrays = {
|
||||||
|
"point_offsets": np.asarray([0, 10, 20], dtype="<i8"),
|
||||||
|
"point_capture_sequence": np.asarray([1, 3], dtype="<i8"),
|
||||||
|
"pose_received_monotonic_ns": np.asarray(
|
||||||
|
[1_001_000_000, 1_101_000_000],
|
||||||
|
dtype="<i8",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def point_frame_count(self) -> int:
|
||||||
|
return 2
|
||||||
|
|
||||||
|
@property
|
||||||
|
def pose_frame_count(self) -> int:
|
||||||
|
return 2
|
||||||
|
|
||||||
|
@property
|
||||||
|
def point_count(self) -> int:
|
||||||
|
return 20
|
||||||
|
|
||||||
|
def point_frame(self, index: int) -> LidarReplayPointFrame:
|
||||||
|
xyz = self._points[index]
|
||||||
|
return LidarReplayPointFrame(
|
||||||
|
capture_sequence=1 + index * 2,
|
||||||
|
payload_bytes=100,
|
||||||
|
received_at_epoch_ns=2_000_000_000 + index * 100_000_000,
|
||||||
|
received_monotonic_ns=1_000_000_000 + index * 100_000_000,
|
||||||
|
header_seq=10 + index,
|
||||||
|
header_stamp=100 + index,
|
||||||
|
scaler=1000,
|
||||||
|
raw_xyz=(xyz * 1000).astype(np.int64),
|
||||||
|
xyz_map=xyz,
|
||||||
|
rgbi=np.full(10, 0xFFFFFF80, dtype=np.uint32),
|
||||||
|
intensity=np.full(10, 128, dtype=np.uint8),
|
||||||
|
)
|
||||||
|
|
||||||
|
def pose_frame(self, index: int) -> LidarReplayPoseFrame:
|
||||||
|
return LidarReplayPoseFrame(
|
||||||
|
capture_sequence=2 + index * 2,
|
||||||
|
payload_bytes=80,
|
||||||
|
received_at_epoch_ns=2_001_000_000 + index * 100_000_000,
|
||||||
|
received_monotonic_ns=1_001_000_000 + index * 100_000_000,
|
||||||
|
header_seq=20 + index,
|
||||||
|
header_stamp=200 + index,
|
||||||
|
header_scaler=1000,
|
||||||
|
pose_stamp=201 + index,
|
||||||
|
position_map=(0.0, 0.0, 0.0),
|
||||||
|
orientation_map_from_lidar=(0.0, 0.0, 0.0, 1.0),
|
||||||
|
distance=0.0,
|
||||||
|
pose_accuracy=0.001,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Candidate:
|
||||||
|
@property
|
||||||
|
def identity(self) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"provider_id": "test-patchwork/v1",
|
||||||
|
"binary_sha256": "c" * 64,
|
||||||
|
}
|
||||||
|
|
||||||
|
def segment(self, xyzi: np.ndarray) -> GroundSegmentation:
|
||||||
|
ground = xyzi[:, 2] <= 0.025
|
||||||
|
assigned = np.ones(xyzi.shape[0], dtype=np.bool_)
|
||||||
|
return GroundSegmentation(ground, assigned, 0.5)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint(router: APIRouter, path: str) -> object:
|
||||||
|
for route in router.routes:
|
||||||
|
if isinstance(route, APIRoute) and route.path == path and "GET" in route.methods:
|
||||||
|
return route.endpoint
|
||||||
|
raise AssertionError(f"GET {path} route is missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_percentile_ground_is_point_aligned_and_non_mutating() -> None:
|
||||||
|
replay = _Replay()
|
||||||
|
xyz = replay._points[0]
|
||||||
|
xyzi = np.column_stack((xyz, np.ones(xyz.shape[0]))).astype(np.float32)
|
||||||
|
unchanged = xyzi.copy()
|
||||||
|
|
||||||
|
result = LocalPercentileGroundSegmenter(
|
||||||
|
profile=DEFAULT_GROUND_BENCHMARK_PROFILE
|
||||||
|
).segment(xyzi)
|
||||||
|
|
||||||
|
assert result.ground_mask.shape == (10,)
|
||||||
|
assert result.assigned_mask.all()
|
||||||
|
assert 0 < np.count_nonzero(result.ground_mask) < 10
|
||||||
|
np.testing.assert_array_equal(xyzi, unchanged)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ground_benchmark_is_immutable_diagnostic_evidence(tmp_path: Path) -> None:
|
||||||
|
replay = _Replay()
|
||||||
|
output = build_lidar_ground_benchmark(
|
||||||
|
replay, # type: ignore[arg-type]
|
||||||
|
tmp_path / "benchmarks",
|
||||||
|
patchwork=_Candidate(),
|
||||||
|
)
|
||||||
|
result = LidarGroundBenchmarkV1(output)
|
||||||
|
try:
|
||||||
|
assert result.report["status"] == "diagnostic-only"
|
||||||
|
assert result.report["input_domain"]["accepted"] is False
|
||||||
|
assert result.report["labels"]["metrics_available"] is False
|
||||||
|
assert (
|
||||||
|
result.report["decision"]["status"]
|
||||||
|
== "do-not-promote-on-current-vendor-map"
|
||||||
|
)
|
||||||
|
assert result.arrays["current_ground"].shape == (20,)
|
||||||
|
assert result.arrays["candidate_ground"].shape == (20,)
|
||||||
|
finally:
|
||||||
|
result.close()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
build_lidar_ground_benchmark(
|
||||||
|
replay, # type: ignore[arg-type]
|
||||||
|
tmp_path / "benchmarks",
|
||||||
|
patchwork=_Candidate(),
|
||||||
|
)
|
||||||
|
== output
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest_path = output / "manifest.json"
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
manifest["identity"]["points"] = 21
|
||||||
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||||
|
with pytest.raises(LidarGroundError, match="identity"):
|
||||||
|
LidarGroundBenchmarkV1(output)
|
||||||
|
|
||||||
|
|
||||||
|
def test_annotation_template_starts_all_ignore_and_never_ground_truth(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
replay = _Replay()
|
||||||
|
output = build_lidar_ground_annotation_template(
|
||||||
|
replay, # type: ignore[arg-type]
|
||||||
|
tmp_path / "annotations",
|
||||||
|
requested_frames=2,
|
||||||
|
)
|
||||||
|
manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8"))
|
||||||
|
arrays = np.load(output / "labels-template.npz", allow_pickle=False)
|
||||||
|
try:
|
||||||
|
assert manifest["ground_truth"] is False
|
||||||
|
assert manifest["identity"]["review_status"] == "unreviewed"
|
||||||
|
assert arrays["labels"].shape == (20,)
|
||||||
|
assert not np.any(arrays["labels"])
|
||||||
|
finally:
|
||||||
|
arrays.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_ground_api_is_read_only_path_free_and_pack_filtered(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
replay = _Replay()
|
||||||
|
root = tmp_path / "benchmarks"
|
||||||
|
output = build_lidar_ground_benchmark(
|
||||||
|
replay, # type: ignore[arg-type]
|
||||||
|
root,
|
||||||
|
patchwork=_Candidate(),
|
||||||
|
)
|
||||||
|
router = build_lidar_router(
|
||||||
|
root_provider=lambda: None,
|
||||||
|
ground_root_provider=lambda: root,
|
||||||
|
)
|
||||||
|
catalog_route = _endpoint(router, "/api/v1/lidar/ground-benchmarks")
|
||||||
|
detail_route = _endpoint(
|
||||||
|
router,
|
||||||
|
"/api/v1/lidar/ground-benchmarks/{benchmark_id}",
|
||||||
|
)
|
||||||
|
|
||||||
|
catalog = catalog_route( # type: ignore[operator]
|
||||||
|
pack_id=replay.pack_id,
|
||||||
|
limit=20,
|
||||||
|
)
|
||||||
|
detail = detail_route(benchmark_id=output.name) # type: ignore[operator]
|
||||||
|
|
||||||
|
assert (
|
||||||
|
catalog["schema_version"]
|
||||||
|
== "missioncore.lidar-ground-benchmark-catalog/v1"
|
||||||
|
)
|
||||||
|
assert catalog["valid_total"] == 1
|
||||||
|
assert catalog["items"][0]["decision"]["production_promotion"] is False
|
||||||
|
assert detail["benchmark"]["benchmark_id"] == output.name
|
||||||
|
assert detail["access"] == "read-only"
|
||||||
|
assert str(tmp_path) not in repr({"catalog": catalog, "detail": detail})
|
||||||
|
|
||||||
|
|
||||||
|
def test_reviewed_ground_metrics_keep_ignore_out_of_denominators() -> None:
|
||||||
|
prediction = GroundSegmentation(
|
||||||
|
ground_mask=np.asarray([True, True, False, False, False, False]),
|
||||||
|
assigned_mask=np.asarray([True, True, True, True, False, True]),
|
||||||
|
latency_ms=1.0,
|
||||||
|
)
|
||||||
|
labels = np.asarray([1, 2, 2, 3, 4, 0], dtype=np.uint8)
|
||||||
|
|
||||||
|
metrics = score_ground_labels(prediction, labels)
|
||||||
|
|
||||||
|
assert metrics["reviewed_points"] == 5
|
||||||
|
assert metrics["ground_iou"] == pytest.approx(0.5)
|
||||||
|
assert metrics["curb_recall"] == pytest.approx(0.5)
|
||||||
|
assert metrics["low_obstacle_recall"] == pytest.approx(1.0)
|
||||||
|
assert metrics["reflection_noise_rejection"] == pytest.approx(1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_patchwork_adapter_records_binary_and_rejects_overlapping_indices(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
binary = tmp_path / "pypatchworkpp.so"
|
||||||
|
binary.write_bytes(b"synthetic-binding")
|
||||||
|
|
||||||
|
class Parameters:
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Estimator:
|
||||||
|
def __init__(self, _params: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def estimateGround(self, _points: np.ndarray) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def getGroundIndices(self) -> list[int]:
|
||||||
|
return [0, 1]
|
||||||
|
|
||||||
|
def getNongroundIndices(self) -> list[int]:
|
||||||
|
return [1, 2]
|
||||||
|
|
||||||
|
module = SimpleNamespace(
|
||||||
|
__file__=str(binary),
|
||||||
|
__version__="test",
|
||||||
|
Parameters=Parameters,
|
||||||
|
patchworkpp=Estimator,
|
||||||
|
)
|
||||||
|
adapter = PatchworkPPGroundSegmenter( # type: ignore[arg-type]
|
||||||
|
module,
|
||||||
|
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||||
|
)
|
||||||
|
assert adapter.identity["binary_sha256"]
|
||||||
|
with pytest.raises(LidarGroundError, match="assigned one point twice"):
|
||||||
|
adapter.segment(np.zeros((3, 4), dtype=np.float32))
|
||||||
Loading…
Reference in New Issue