feat: qualify complete RELLIS ground dataset
This commit is contained in:
parent
054feec0d2
commit
f2f044080f
|
|
@ -12,7 +12,8 @@ export type DatasetSourceAdmissionStatus =
|
||||||
| "verifying"
|
| "verifying"
|
||||||
| "verified"
|
| "verified"
|
||||||
| "frame-ready"
|
| "frame-ready"
|
||||||
| "smoke-ready";
|
| "smoke-ready"
|
||||||
|
| "dataset-ready";
|
||||||
|
|
||||||
export interface DatasetSource {
|
export interface DatasetSource {
|
||||||
sourceId: string;
|
sourceId: string;
|
||||||
|
|
@ -308,6 +309,7 @@ export function parseDatasetGatewayCatalog(
|
||||||
&& admissionStatus !== "verified"
|
&& admissionStatus !== "verified"
|
||||||
&& admissionStatus !== "frame-ready"
|
&& admissionStatus !== "frame-ready"
|
||||||
&& admissionStatus !== "smoke-ready"
|
&& admissionStatus !== "smoke-ready"
|
||||||
|
&& admissionStatus !== "dataset-ready"
|
||||||
) {
|
) {
|
||||||
throw new DatasetGatewayContractError("source admission status неизвестен");
|
throw new DatasetGatewayContractError("source admission status неизвестен");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,10 @@ export interface GroundQualification {
|
||||||
checks: QualificationCheck[];
|
checks: QualificationCheck[];
|
||||||
worstFrames: QualificationWorstFrame[];
|
worstFrames: QualificationWorstFrame[];
|
||||||
decision: {
|
decision: {
|
||||||
status: "shadow-candidate" | "qualification-rejected";
|
status:
|
||||||
|
| "shadow-candidate"
|
||||||
|
| "cross-dataset-qualified"
|
||||||
|
| "qualification-rejected";
|
||||||
passed: boolean;
|
passed: boolean;
|
||||||
promotedToNavigationOrSafety: false;
|
promotedToNavigationOrSafety: false;
|
||||||
reason: string;
|
reason: string;
|
||||||
|
|
@ -75,7 +78,7 @@ export interface GroundReviewFrameSummary {
|
||||||
frameId: string;
|
frameId: string;
|
||||||
datasetSequenceId: string;
|
datasetSequenceId: string;
|
||||||
datasetFrameNumber: number;
|
datasetFrameNumber: number;
|
||||||
sensorTimestampNs: string;
|
sensorTimestampNs: string | null;
|
||||||
sourcePointCount: number;
|
sourcePointCount: number;
|
||||||
pointCount: number;
|
pointCount: number;
|
||||||
current: GroundReviewFrameMetrics;
|
current: GroundReviewFrameMetrics;
|
||||||
|
|
@ -264,7 +267,11 @@ export function decodeGroundQualification(payload: unknown): GroundQualification
|
||||||
);
|
);
|
||||||
const decision = record(source.decision, "decision");
|
const decision = record(source.decision, "decision");
|
||||||
const status = text(decision.status, "decision.status", 64);
|
const status = text(decision.status, "decision.status", 64);
|
||||||
if (status !== "shadow-candidate" && status !== "qualification-rejected") {
|
if (
|
||||||
|
status !== "shadow-candidate"
|
||||||
|
&& status !== "cross-dataset-qualified"
|
||||||
|
&& status !== "qualification-rejected"
|
||||||
|
) {
|
||||||
throw new GroundQualificationContractError("Решение квалификации неизвестно.");
|
throw new GroundQualificationContractError("Решение квалификации неизвестно.");
|
||||||
}
|
}
|
||||||
if (decision.promoted_to_navigation_or_safety !== false) {
|
if (decision.promoted_to_navigation_or_safety !== false) {
|
||||||
|
|
@ -384,7 +391,11 @@ export function decodeGroundReview(payload: unknown): GroundReview {
|
||||||
throw new GroundQualificationContractError("Неизвестная схема покадрового просмотра.");
|
throw new GroundQualificationContractError("Неизвестная схема покадрового просмотра.");
|
||||||
}
|
}
|
||||||
const frameCount = integer(source.frame_count, "frame_count", 1);
|
const frameCount = integer(source.frame_count, "frame_count", 1);
|
||||||
const frames = array(source.frames, "frames", 2_000).map(
|
const sourceId = text(source.source_id, "source_id");
|
||||||
|
if (sourceId !== "goose-3d/v2025-08-22" && sourceId !== "rellis-3d/v1.1") {
|
||||||
|
throw new GroundQualificationContractError("Источник review неизвестен.");
|
||||||
|
}
|
||||||
|
const frames = array(source.frames, "frames", 3_000).map(
|
||||||
(value, index): GroundReviewFrameSummary => {
|
(value, index): GroundReviewFrameSummary => {
|
||||||
const frame = record(value, `frames[${index}]`);
|
const frame = record(value, `frames[${index}]`);
|
||||||
const sequence = integer(frame.sequence, `frames[${index}].sequence`);
|
const sequence = integer(frame.sequence, `frames[${index}].sequence`);
|
||||||
|
|
@ -392,7 +403,6 @@ export function decodeGroundReview(payload: unknown): GroundReview {
|
||||||
throw new GroundQualificationContractError("Кадры просмотра идут не по порядку.");
|
throw new GroundQualificationContractError("Кадры просмотра идут не по порядку.");
|
||||||
}
|
}
|
||||||
const frameId = id(frame.frame_id, `frames[${index}].frame_id`);
|
const frameId = id(frame.frame_id, `frames[${index}].frame_id`);
|
||||||
const frameIdentity = parseGooseFrameId(frameId);
|
|
||||||
const datasetSequenceId = id(
|
const datasetSequenceId = id(
|
||||||
frame.dataset_sequence_id,
|
frame.dataset_sequence_id,
|
||||||
`frames[${index}].dataset_sequence_id`,
|
`frames[${index}].dataset_sequence_id`,
|
||||||
|
|
@ -401,7 +411,10 @@ export function decodeGroundReview(payload: unknown): GroundReview {
|
||||||
frame.dataset_frame_number,
|
frame.dataset_frame_number,
|
||||||
`frames[${index}].dataset_frame_number`,
|
`frames[${index}].dataset_frame_number`,
|
||||||
);
|
);
|
||||||
const sensorTimestampNs = text(
|
let sensorTimestampNs: string | null = null;
|
||||||
|
if (sourceId === "goose-3d/v2025-08-22") {
|
||||||
|
const frameIdentity = parseGooseFrameId(frameId);
|
||||||
|
sensorTimestampNs = text(
|
||||||
frame.sensor_timestamp_ns,
|
frame.sensor_timestamp_ns,
|
||||||
`frames[${index}].sensor_timestamp_ns`,
|
`frames[${index}].sensor_timestamp_ns`,
|
||||||
20,
|
20,
|
||||||
|
|
@ -415,6 +428,15 @@ export function decodeGroundReview(payload: unknown): GroundReview {
|
||||||
`frames[${index}] противоречит идентичности GOOSE-кадра.`,
|
`frames[${index}] противоречит идентичности GOOSE-кадра.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else if (
|
||||||
|
frame.sensor_timestamp_ns !== null
|
||||||
|
|| !/^rellis-0000[0-4]-[0-9]{6}$/.test(frameId)
|
||||||
|
|| frameId !== `rellis-${datasetSequenceId}-${String(datasetFrameNumber).padStart(6, "0")}`
|
||||||
|
) {
|
||||||
|
throw new GroundQualificationContractError(
|
||||||
|
`frames[${index}] противоречит идентичности RELLIS-кадра.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
sequence,
|
sequence,
|
||||||
frameId,
|
frameId,
|
||||||
|
|
@ -457,7 +479,7 @@ export function decodeGroundReview(payload: unknown): GroundReview {
|
||||||
return {
|
return {
|
||||||
runId: id(source.run_id, "run_id"),
|
runId: id(source.run_id, "run_id"),
|
||||||
identitySha256,
|
identitySha256,
|
||||||
sourceId: text(source.source_id, "source_id"),
|
sourceId,
|
||||||
frameCount,
|
frameCount,
|
||||||
previewPoints: integer(source.preview_points, "preview_points", 1),
|
previewPoints: integer(source.preview_points, "preview_points", 1),
|
||||||
frames,
|
frames,
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,7 @@ function admissionLabel(status: DatasetSourceAdmissionStatus): string {
|
||||||
verified: "Проверен",
|
verified: "Проверен",
|
||||||
"frame-ready": "Dataset готов",
|
"frame-ready": "Dataset готов",
|
||||||
"smoke-ready": "Smoke готов",
|
"smoke-ready": "Smoke готов",
|
||||||
|
"dataset-ready": "Dataset готов",
|
||||||
};
|
};
|
||||||
return labels[status];
|
return labels[status];
|
||||||
}
|
}
|
||||||
|
|
@ -60,9 +61,17 @@ function isGooseQualificationRun(
|
||||||
&& item.scenarioGeneration.startsWith("goose-3d");
|
&& item.scenarioGeneration.startsWith("goose-3d");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isRellisQualificationRun(
|
||||||
|
item: PolygonRunCatalog["items"][number],
|
||||||
|
): boolean {
|
||||||
|
return item.kind === "replay_shadow"
|
||||||
|
&& item.scenarioGeneration.startsWith("rellis-3d");
|
||||||
|
}
|
||||||
|
|
||||||
function sourceReady(source: DatasetSource): boolean {
|
function sourceReady(source: DatasetSource): boolean {
|
||||||
return source.admissionStatus === "frame-ready"
|
return source.admissionStatus === "frame-ready"
|
||||||
|| source.admissionStatus === "smoke-ready";
|
|| source.admissionStatus === "smoke-ready"
|
||||||
|
|| source.admissionStatus === "dataset-ready";
|
||||||
}
|
}
|
||||||
|
|
||||||
function sourceDescription(source: DatasetSource): string {
|
function sourceDescription(source: DatasetSource): string {
|
||||||
|
|
@ -76,6 +85,9 @@ function sourceFooter(source: DatasetSource, storageReady: boolean): string {
|
||||||
if (source.admissionStatus === "frame-ready") {
|
if (source.admissionStatus === "frame-ready") {
|
||||||
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
|
return "Native scans и разметка приняты. Можно открыть фрагменты и результаты алгоритмов.";
|
||||||
}
|
}
|
||||||
|
if (source.admissionStatus === "dataset-ready") {
|
||||||
|
return "Все официальные scans, labels, poses и split-индексы выровнены на worker D.";
|
||||||
|
}
|
||||||
if (source.admissionStatus === "smoke-ready") {
|
if (source.admissionStatus === "smoke-ready") {
|
||||||
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
|
return "Официальный пример совместим. Можно проверить scan, классы и ground mapping.";
|
||||||
}
|
}
|
||||||
|
|
@ -98,6 +110,7 @@ export function DatasetGatewayWorkspace() {
|
||||||
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
|
const [openSourceId, setOpenSourceId] = useState<string | null>(null);
|
||||||
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
|
const [runCatalog, setRunCatalog] = useState<PolygonRunCatalog | null>(null);
|
||||||
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
|
const [selectedRunId, setSelectedRunId] = useState<string | null>(null);
|
||||||
|
const [selectedRellisRunId, setSelectedRellisRunId] = useState<string | null>(null);
|
||||||
const [runError, setRunError] = useState<string | null>(null);
|
const [runError, setRunError] = useState<string | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
@ -143,6 +156,11 @@ export function DatasetGatewayWorkspace() {
|
||||||
?? gooseRuns[0]
|
?? gooseRuns[0]
|
||||||
?? null;
|
?? null;
|
||||||
setSelectedRunId(selected?.runId ?? null);
|
setSelectedRunId(selected?.runId ?? null);
|
||||||
|
const rellisRuns = value.items.filter(isRellisQualificationRun);
|
||||||
|
const selectedRellis = rellisRuns.find((item) => item.runId === requested)
|
||||||
|
?? rellisRuns[0]
|
||||||
|
?? null;
|
||||||
|
setSelectedRellisRunId(selectedRellis?.runId ?? null);
|
||||||
})
|
})
|
||||||
.catch((loadError: unknown) => {
|
.catch((loadError: unknown) => {
|
||||||
if (!controller.signal.aborted) setRunError(errorMessage(loadError));
|
if (!controller.signal.aborted) setRunError(errorMessage(loadError));
|
||||||
|
|
@ -151,6 +169,7 @@ export function DatasetGatewayWorkspace() {
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
|
const gooseRuns = runCatalog?.items.filter(isGooseQualificationRun) ?? [];
|
||||||
|
const rellisRuns = runCatalog?.items.filter(isRellisQualificationRun) ?? [];
|
||||||
const openSource = catalog?.sources.find(
|
const openSource = catalog?.sources.find(
|
||||||
(source) => source.sourceId === openSourceId,
|
(source) => source.sourceId === openSourceId,
|
||||||
) ?? null;
|
) ?? null;
|
||||||
|
|
@ -322,7 +341,28 @@ export function DatasetGatewayWorkspace() {
|
||||||
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
|
<p>{runError ?? "Нет совместимой версии покадрового анализа."}</p>
|
||||||
</section>
|
</section>
|
||||||
) : openSource?.sourceKind === "rellis" ? (
|
) : openSource?.sourceKind === "rellis" ? (
|
||||||
<RellisDatasetReview />
|
<>
|
||||||
|
{rellisRuns.length > 1 && selectedRellisRunId ? (
|
||||||
|
<div className="dataset-result-select">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">РЕЗУЛЬТАТЫ ОБРАБОТКИ</span>
|
||||||
|
<strong>Версия анализа</strong>
|
||||||
|
</div>
|
||||||
|
<select
|
||||||
|
aria-label="Выбрать версию анализа RELLIS"
|
||||||
|
value={selectedRellisRunId}
|
||||||
|
onChange={(event) => setSelectedRellisRunId(event.target.value)}
|
||||||
|
>
|
||||||
|
{rellisRuns.map((item) => (
|
||||||
|
<option key={item.runId} value={item.runId}>
|
||||||
|
{item.profileGeneration}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
<RellisDatasetReview runId={selectedRellisRunId} />
|
||||||
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<details className="dataset-contract">
|
<details className="dataset-contract">
|
||||||
|
|
|
||||||
|
|
@ -68,6 +68,9 @@ function secondsBetween(
|
||||||
left: GroundReviewFrameSummary,
|
left: GroundReviewFrameSummary,
|
||||||
right: GroundReviewFrameSummary,
|
right: GroundReviewFrameSummary,
|
||||||
): number {
|
): number {
|
||||||
|
if (left.sensorTimestampNs === null || right.sensorTimestampNs === null) {
|
||||||
|
throw new Error("GOOSE frame timestamp отсутствует.");
|
||||||
|
}
|
||||||
const delta = BigInt(right.sensorTimestampNs) - BigInt(left.sensorTimestampNs);
|
const delta = BigInt(right.sensorTimestampNs) - BigInt(left.sensorTimestampNs);
|
||||||
return Number(delta) / 1_000_000_000;
|
return Number(delta) / 1_000_000_000;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { useEffect, useMemo, useState } from "react";
|
import { useEffect, useMemo, useRef, useState } from "react";
|
||||||
import { StatusBadge } from "@nodedc/ui-react";
|
import { StatusBadge } from "@nodedc/ui-react";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
@ -9,6 +9,13 @@ import {
|
||||||
LidarGroundPointCloud,
|
LidarGroundPointCloud,
|
||||||
type LidarGroundViewMode,
|
type LidarGroundViewMode,
|
||||||
} from "./LidarGroundPointCloud";
|
} from "./LidarGroundPointCloud";
|
||||||
|
import {
|
||||||
|
fetchGroundReview,
|
||||||
|
fetchGroundReviewFrame,
|
||||||
|
type GroundReview,
|
||||||
|
type GroundReviewFrame,
|
||||||
|
type GroundReviewFrameSummary,
|
||||||
|
} from "../core/polygon/groundQualification";
|
||||||
|
|
||||||
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
|
const modes: ReadonlyArray<[LidarGroundViewMode, string]> = [
|
||||||
["intensity", "Исходный скан"],
|
["intensity", "Исходный скан"],
|
||||||
|
|
@ -39,7 +46,25 @@ function modeExplanation(mode: LidarGroundViewMode): string {
|
||||||
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
|
return "Один исходный оборот Ouster OS1 64; цвет — нормализованная remission.";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function RellisDatasetReview() {
|
interface RellisDatasetReviewProps {
|
||||||
|
runId?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const qualifiedModes: ReadonlyArray<[LidarGroundViewMode, string]> = [
|
||||||
|
["intensity", "Исходный скан"],
|
||||||
|
["ground-truth", "Ground truth"],
|
||||||
|
["current", "Текущий алгоритм"],
|
||||||
|
["candidate", "Patchwork++"],
|
||||||
|
["disagreement", "Ошибки текущего"],
|
||||||
|
["candidate-disagreement", "Ошибки Patchwork++"],
|
||||||
|
];
|
||||||
|
|
||||||
|
export function RellisDatasetReview({ runId = null }: RellisDatasetReviewProps) {
|
||||||
|
if (runId) return <RellisQualifiedReview runId={runId} />;
|
||||||
|
return <RellisSmokeReview />;
|
||||||
|
}
|
||||||
|
|
||||||
|
function RellisSmokeReview() {
|
||||||
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
|
const [preview, setPreview] = useState<DatasetNativeScanPreview | null>(null);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
|
const [mode, setMode] = useState<LidarGroundViewMode>("semantic");
|
||||||
|
|
@ -237,3 +262,313 @@ export function RellisDatasetReview() {
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function reviewScene(frame: GroundReviewFrame | null) {
|
||||||
|
if (!frame) return null;
|
||||||
|
return {
|
||||||
|
pointCount: frame.pointCount,
|
||||||
|
pointsXyzM: frame.pointsXyzM,
|
||||||
|
intensity0To255: frame.intensity0To255,
|
||||||
|
masks: {
|
||||||
|
currentGround: frame.currentGround,
|
||||||
|
currentAssigned: frame.evaluated,
|
||||||
|
candidateGround: frame.patchworkGround,
|
||||||
|
candidateAssigned: frame.evaluated,
|
||||||
|
disagreement: frame.currentDisagreement,
|
||||||
|
candidateDisagreement: frame.patchworkDisagreement,
|
||||||
|
groundTruthGround: frame.groundTruthGround,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function groupRellisFrames(review: GroundReview) {
|
||||||
|
const groups = new Map<string, GroundReviewFrameSummary[]>();
|
||||||
|
review.frames.forEach((frame) => {
|
||||||
|
const frames = groups.get(frame.datasetSequenceId) ?? [];
|
||||||
|
frames.push(frame);
|
||||||
|
groups.set(frame.datasetSequenceId, frames);
|
||||||
|
});
|
||||||
|
return [...groups.entries()].map(([sequenceId, frames]) => ({
|
||||||
|
sequenceId,
|
||||||
|
frames,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function averageMetric(
|
||||||
|
frames: GroundReviewFrameSummary[],
|
||||||
|
value: (frame: GroundReviewFrameSummary) => number,
|
||||||
|
) {
|
||||||
|
return frames.reduce((total, frame) => total + value(frame), 0) / frames.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function qualifiedModeExplanation(mode: LidarGroundViewMode) {
|
||||||
|
if (mode === "intensity") return "Один исходный Ouster OS1 scan; цвет — remission.";
|
||||||
|
if (mode === "ground-truth") return "Point-aligned ground target из RELLIS labels.";
|
||||||
|
if (mode === "current") return "Результат текущего Mission Core baseline.";
|
||||||
|
if (mode === "candidate") return "Результат закреплённого Patchwork++ v1.4.1.";
|
||||||
|
if (mode === "disagreement") return "Красным — ошибки текущего baseline.";
|
||||||
|
return "Красным — ошибки Patchwork++ относительно labels.";
|
||||||
|
}
|
||||||
|
|
||||||
|
function RellisQualifiedReview({ runId }: { runId: string }) {
|
||||||
|
const [review, setReview] = useState<GroundReview | null>(null);
|
||||||
|
const [frame, setFrame] = useState<GroundReviewFrame | null>(null);
|
||||||
|
const [selectedSequence, setSelectedSequence] = useState<string | null>(null);
|
||||||
|
const [selectedFrame, setSelectedFrame] = useState<string | null>(null);
|
||||||
|
const [mode, setMode] = useState<LidarGroundViewMode>("intensity");
|
||||||
|
const [playing, setPlaying] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const cache = useRef(new Map<string, GroundReviewFrame>());
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
setError(null);
|
||||||
|
setReview(null);
|
||||||
|
setFrame(null);
|
||||||
|
cache.current.clear();
|
||||||
|
void fetchGroundReview(runId, { signal: controller.signal })
|
||||||
|
.then((value) => {
|
||||||
|
if (controller.signal.aborted || !value) return;
|
||||||
|
setReview(value);
|
||||||
|
setSelectedSequence(value.frames[0]?.datasetSequenceId ?? null);
|
||||||
|
setSelectedFrame(value.frames[0]?.frameId ?? null);
|
||||||
|
})
|
||||||
|
.catch((loadError: unknown) => {
|
||||||
|
if (!controller.signal.aborted) setError(errorMessage(loadError));
|
||||||
|
});
|
||||||
|
return () => controller.abort();
|
||||||
|
}, [runId]);
|
||||||
|
|
||||||
|
const sequences = useMemo(
|
||||||
|
() => review ? groupRellisFrames(review) : [],
|
||||||
|
[review],
|
||||||
|
);
|
||||||
|
const sequence = sequences.find((item) => item.sequenceId === selectedSequence)
|
||||||
|
?? sequences[0]
|
||||||
|
?? null;
|
||||||
|
const position = Math.max(
|
||||||
|
0,
|
||||||
|
sequence?.frames.findIndex((item) => item.frameId === selectedFrame) ?? 0,
|
||||||
|
);
|
||||||
|
const summary = sequence?.frames[position] ?? null;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!selectedFrame) return;
|
||||||
|
const cached = cache.current.get(selectedFrame);
|
||||||
|
if (cached) {
|
||||||
|
setFrame(cached);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const controller = new AbortController();
|
||||||
|
void fetchGroundReviewFrame(runId, selectedFrame, { signal: controller.signal })
|
||||||
|
.then((value) => {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
cache.current.set(value.frameId, value);
|
||||||
|
while (cache.current.size > 32) {
|
||||||
|
const first = cache.current.keys().next().value as string | undefined;
|
||||||
|
if (first) cache.current.delete(first);
|
||||||
|
else break;
|
||||||
|
}
|
||||||
|
setFrame(value);
|
||||||
|
})
|
||||||
|
.catch((loadError: unknown) => {
|
||||||
|
if (!controller.signal.aborted) setError(errorMessage(loadError));
|
||||||
|
});
|
||||||
|
return () => controller.abort();
|
||||||
|
}, [runId, selectedFrame]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!playing || !sequence || frame?.frameId !== selectedFrame) return;
|
||||||
|
const timer = window.setTimeout(() => {
|
||||||
|
const next = sequence.frames[position + 1];
|
||||||
|
if (!next) {
|
||||||
|
setPlaying(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSelectedFrame(next.frameId);
|
||||||
|
}, 250);
|
||||||
|
return () => window.clearTimeout(timer);
|
||||||
|
}, [frame?.frameId, playing, position, selectedFrame, sequence]);
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<section className="polygon-review-unavailable">
|
||||||
|
<StatusBadge tone="danger">Review недоступен</StatusBadge>
|
||||||
|
<h3>RELLIS-3D не открыт</h3>
|
||||||
|
<p>{error}</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!review || !sequence) {
|
||||||
|
return (
|
||||||
|
<section className="polygon-review-unavailable">
|
||||||
|
<StatusBadge tone="accent">Читаем индекс</StatusBadge>
|
||||||
|
<h3>Открываем RELLIS validation</h3>
|
||||||
|
<p>Исходные архивы остаются на worker D; в браузер идёт bounded preview.</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const selectPosition = (value: number) => {
|
||||||
|
const bounded = Math.max(0, Math.min(value, sequence.frames.length - 1));
|
||||||
|
setPlaying(false);
|
||||||
|
setSelectedFrame(sequence.frames[bounded]?.frameId ?? null);
|
||||||
|
};
|
||||||
|
const patchworkBetter = sequence.frames.filter(
|
||||||
|
(item) => item.groundIouDelta > 0,
|
||||||
|
).length;
|
||||||
|
const scene = reviewScene(frame);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section
|
||||||
|
className="dataset-sequence-review rellis-review"
|
||||||
|
aria-label="Просмотр RELLIS validation"
|
||||||
|
>
|
||||||
|
<header className="dataset-sequence-review__truth">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">RELLIS · FULL VALIDATION</span>
|
||||||
|
<h2>Пять последовательностей Ouster OS1 с покадровым сравнением</h2>
|
||||||
|
<p>
|
||||||
|
Все {review.frameCount.toLocaleString("ru-RU")} validation-сканов
|
||||||
|
выровнены с labels и pose index. Высота Patchwork++ определена только
|
||||||
|
на train split; validation labels использованы исключительно для оценки.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<dl>
|
||||||
|
<div><dt>Последовательностей</dt><dd>{sequences.length}</dd></div>
|
||||||
|
<div><dt>Validation scans</dt><dd>{review.frameCount}</dd></div>
|
||||||
|
<div><dt>Preview</dt><dd>{review.previewPoints.toLocaleString("ru-RU")} точек</dd></div>
|
||||||
|
</dl>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div className="dataset-sequence-select">
|
||||||
|
<label htmlFor="rellis-sequence">Последовательность датасета</label>
|
||||||
|
<select
|
||||||
|
id="rellis-sequence"
|
||||||
|
value={sequence.sequenceId}
|
||||||
|
onChange={(event) => {
|
||||||
|
const next = sequences.find((item) => item.sequenceId === event.target.value);
|
||||||
|
setPlaying(false);
|
||||||
|
setSelectedSequence(event.target.value);
|
||||||
|
setSelectedFrame(next?.frames[0]?.frameId ?? null);
|
||||||
|
setFrame(null);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{sequences.map((item) => (
|
||||||
|
<option key={item.sequenceId} value={item.sequenceId}>
|
||||||
|
Sequence {item.sequenceId} · {item.frames.length} scans
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<span>
|
||||||
|
Индексы {sequence.frames[0]?.datasetFrameNumber ?? 0}–{
|
||||||
|
sequence.frames[sequence.frames.length - 1]?.datasetFrameNumber ?? 0
|
||||||
|
}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="polygon-review-player">
|
||||||
|
<header className="polygon-review-player__header">
|
||||||
|
<div>
|
||||||
|
<span className="section-eyebrow">
|
||||||
|
SCAN {position + 1} / {sequence.frames.length}
|
||||||
|
{" · SOURCE FRAME "}{summary?.datasetFrameNumber ?? "—"}
|
||||||
|
</span>
|
||||||
|
<strong>{summary?.frameId ?? "Загружаем scan"}</strong>
|
||||||
|
</div>
|
||||||
|
<div className="polygon-review-mode" aria-label="Режим RELLIS review">
|
||||||
|
{qualifiedModes.map(([viewMode, label]) => (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
key={viewMode}
|
||||||
|
data-active={mode === viewMode ? "true" : undefined}
|
||||||
|
onClick={() => setMode(viewMode)}
|
||||||
|
>
|
||||||
|
{label}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
<div className="polygon-review-controls">
|
||||||
|
<button type="button" onClick={() => setPlaying((value) => !value)}>
|
||||||
|
{playing ? "Пауза" : "Play sequence"}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
disabled={position === 0}
|
||||||
|
onClick={() => selectPosition(position - 1)}
|
||||||
|
>
|
||||||
|
←
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
disabled={position === sequence.frames.length - 1}
|
||||||
|
onClick={() => selectPosition(position + 1)}
|
||||||
|
>
|
||||||
|
→
|
||||||
|
</button>
|
||||||
|
<input
|
||||||
|
type="range"
|
||||||
|
min={0}
|
||||||
|
max={sequence.frames.length - 1}
|
||||||
|
value={position}
|
||||||
|
aria-label="Позиция в RELLIS sequence"
|
||||||
|
onChange={(event) => selectPosition(Number(event.target.value))}
|
||||||
|
/>
|
||||||
|
<span>{position + 1} / {sequence.frames.length}</span>
|
||||||
|
</div>
|
||||||
|
<div className="polygon-review-stage">
|
||||||
|
{scene ? (
|
||||||
|
<LidarGroundPointCloud
|
||||||
|
key={sequence.sequenceId}
|
||||||
|
frame={scene}
|
||||||
|
mode={mode}
|
||||||
|
coordinateFrame="sensor-fixed"
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="polygon-review-stage__loading">Загружаем scan…</div>
|
||||||
|
)}
|
||||||
|
<div className="polygon-review-stage__legend">
|
||||||
|
<strong>
|
||||||
|
{qualifiedModes.find(([viewMode]) => viewMode === mode)?.[1]}
|
||||||
|
</strong>
|
||||||
|
<span>{qualifiedModeExplanation(mode)}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<div className="rellis-review__metrics">
|
||||||
|
<article>
|
||||||
|
<span>Ground IoU · Current</span>
|
||||||
|
<strong>{formatPercent(averageMetric(sequence.frames, (item) => item.current.groundIou))}</strong>
|
||||||
|
<small>среднее по выбранной sequence</small>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Ground IoU · Patchwork++</span>
|
||||||
|
<strong>
|
||||||
|
{formatPercent(averageMetric(sequence.frames, (item) => item.patchwork.groundIou))}
|
||||||
|
</strong>
|
||||||
|
<small>лучше Current на {patchworkBetter} scans</small>
|
||||||
|
</article>
|
||||||
|
<article>
|
||||||
|
<span>Obstacle recall · Patchwork++</span>
|
||||||
|
<strong>
|
||||||
|
{formatPercent(
|
||||||
|
averageMetric(
|
||||||
|
sequence.frames,
|
||||||
|
(item) => item.patchwork.obstacleNonGroundRecall,
|
||||||
|
),
|
||||||
|
)}
|
||||||
|
</strong>
|
||||||
|
<small>non-ground сохранён как препятствие</small>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="dataset-sequence-review__raw-note">
|
||||||
|
Это независимая исследовательская проверка переносимости. Она не
|
||||||
|
включает управление машиной и не принимает алгоритм для navigation/safety.
|
||||||
|
Лицензия исходника — CC-BY-NC-SA-3.0.
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -184,6 +184,40 @@ test("decodes a complete ordered frame review", () => {
|
||||||
assert.equal(decoded.frames[0].patchwork.groundIou, 0.7);
|
assert.equal(decoded.frames[0].patchwork.groundIou, 0.7);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("decodes RELLIS sequences without inventing sensor timestamps", () => {
|
||||||
|
const metric = {
|
||||||
|
ground_iou: 0.6,
|
||||||
|
natural_ground_recall: 0.7,
|
||||||
|
obstacle_non_ground_recall: 0.93,
|
||||||
|
latency_ms: 18,
|
||||||
|
};
|
||||||
|
const decoded = decodeGroundReview({
|
||||||
|
schema_version: "missioncore.polygon-ground-review/v2",
|
||||||
|
access: "read-only",
|
||||||
|
run_id: "rellis-ground-abc",
|
||||||
|
identity_sha256: "c".repeat(64),
|
||||||
|
source_id: "rellis-3d/v1.1",
|
||||||
|
frame_count: 1,
|
||||||
|
preview_points: 12_000,
|
||||||
|
frames: [{
|
||||||
|
sequence: 0,
|
||||||
|
frame_id: "rellis-00000-000307",
|
||||||
|
dataset_sequence_id: "00000",
|
||||||
|
dataset_frame_number: 307,
|
||||||
|
sensor_timestamp_ns: null,
|
||||||
|
source_point_count: 131_072,
|
||||||
|
point_count: 12_000,
|
||||||
|
current: metric,
|
||||||
|
patchworkpp: metric,
|
||||||
|
ground_iou_delta: 0,
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
assert.equal(decoded.sourceId, "rellis-3d/v1.1");
|
||||||
|
assert.equal(decoded.frames[0].datasetSequenceId, "00000");
|
||||||
|
assert.equal(decoded.frames[0].datasetFrameNumber, 307);
|
||||||
|
assert.equal(decoded.frames[0].sensorTimestampNs, null);
|
||||||
|
});
|
||||||
|
|
||||||
test("parses GOOSE sequence, source frame and nanosecond timestamp", () => {
|
test("parses GOOSE sequence, source frame and nanosecond timestamp", () => {
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
parseGooseFrameId("2023-05-17_neubiberg_sunny__0459_1684330012686564560"),
|
parseGooseFrameId("2023-05-17_neubiberg_sunny__0459_1684330012686564560"),
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,22 @@ from k1link.datasets.goose_qualification import (
|
||||||
GroundAcceptancePolicy,
|
GroundAcceptancePolicy,
|
||||||
qualify_goose_ground,
|
qualify_goose_ground,
|
||||||
)
|
)
|
||||||
|
from k1link.datasets.rellis_admission import (
|
||||||
|
RELLIS_ADMISSION_SCHEMA,
|
||||||
|
RellisAdmissionError,
|
||||||
|
admit_rellis_release,
|
||||||
|
)
|
||||||
|
from k1link.datasets.rellis_profile import (
|
||||||
|
RELLIS_PATCHWORK_PROFILE_SCHEMA,
|
||||||
|
RellisPatchworkProfile,
|
||||||
|
)
|
||||||
|
from k1link.datasets.rellis_qualification import (
|
||||||
|
RELLIS_QUALIFICATION_REPORT_SCHEMA,
|
||||||
|
RELLIS_REVIEW_PACK_SCHEMA,
|
||||||
|
RellisAcceptancePolicy,
|
||||||
|
calibrate_rellis_sensor_height,
|
||||||
|
qualify_rellis_ground,
|
||||||
|
)
|
||||||
from k1link.datasets.rellis_smoke import (
|
from k1link.datasets.rellis_smoke import (
|
||||||
RELLIS_CLASSES,
|
RELLIS_CLASSES,
|
||||||
RELLIS_GROUND_POLICY_SCHEMA,
|
RELLIS_GROUND_POLICY_SCHEMA,
|
||||||
|
|
@ -64,9 +80,16 @@ __all__ = [
|
||||||
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
||||||
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
||||||
"RELLIS_CLASSES",
|
"RELLIS_CLASSES",
|
||||||
|
"RELLIS_ADMISSION_SCHEMA",
|
||||||
"RELLIS_GROUND_POLICY_SCHEMA",
|
"RELLIS_GROUND_POLICY_SCHEMA",
|
||||||
|
"RELLIS_PATCHWORK_PROFILE_SCHEMA",
|
||||||
"RELLIS_PREVIEW_SCHEMA",
|
"RELLIS_PREVIEW_SCHEMA",
|
||||||
|
"RELLIS_QUALIFICATION_REPORT_SCHEMA",
|
||||||
|
"RELLIS_REVIEW_PACK_SCHEMA",
|
||||||
"RELLIS_SOURCE_ID",
|
"RELLIS_SOURCE_ID",
|
||||||
|
"RellisAcceptancePolicy",
|
||||||
|
"RellisAdmissionError",
|
||||||
|
"RellisPatchworkProfile",
|
||||||
"RellisSmokeError",
|
"RellisSmokeError",
|
||||||
"GoosePatchworkProfile",
|
"GoosePatchworkProfile",
|
||||||
"GroundAcceptancePolicy",
|
"GroundAcceptancePolicy",
|
||||||
|
|
@ -74,7 +97,9 @@ __all__ = [
|
||||||
"DEFAULT_DEGRADATIONS",
|
"DEFAULT_DEGRADATIONS",
|
||||||
"benchmark_goose_current_ground",
|
"benchmark_goose_current_ground",
|
||||||
"benchmark_goose_patchwork_ground",
|
"benchmark_goose_patchwork_ground",
|
||||||
|
"admit_rellis_release",
|
||||||
"build_rellis_official_smoke_preview",
|
"build_rellis_official_smoke_preview",
|
||||||
|
"calibrate_rellis_sensor_height",
|
||||||
"configured_dataset_admission_manifest",
|
"configured_dataset_admission_manifest",
|
||||||
"configured_dataset_ground_preview",
|
"configured_dataset_ground_preview",
|
||||||
"configured_dataset_preview",
|
"configured_dataset_preview",
|
||||||
|
|
@ -85,5 +110,6 @@ __all__ = [
|
||||||
"read_dataset_native_scan_preview",
|
"read_dataset_native_scan_preview",
|
||||||
"read_semantic_kitti_frame",
|
"read_semantic_kitti_frame",
|
||||||
"rellis_native_scan_preview",
|
"rellis_native_scan_preview",
|
||||||
|
"qualify_rellis_ground",
|
||||||
"qualify_goose_ground",
|
"qualify_goose_ground",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,8 @@ from k1link.datasets.goose_benchmark import (
|
||||||
)
|
)
|
||||||
from k1link.datasets.goose_qualification import qualify_goose_ground
|
from k1link.datasets.goose_qualification import qualify_goose_ground
|
||||||
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
||||||
|
from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release
|
||||||
|
from k1link.datasets.rellis_qualification import qualify_rellis_ground
|
||||||
from k1link.datasets.rellis_smoke import (
|
from k1link.datasets.rellis_smoke import (
|
||||||
RellisSmokeError,
|
RellisSmokeError,
|
||||||
build_rellis_official_smoke_preview,
|
build_rellis_official_smoke_preview,
|
||||||
|
|
@ -207,5 +209,93 @@ def build_rellis_smoke_preview_command(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.command("admit-rellis")
|
||||||
|
def admit_rellis_command(
|
||||||
|
dataset_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
scan_archive: Annotated[
|
||||||
|
Path | None,
|
||||||
|
typer.Option("--scan-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
] = None,
|
||||||
|
label_archive: Annotated[
|
||||||
|
Path | None,
|
||||||
|
typer.Option("--label-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
] = None,
|
||||||
|
pose_archive: Annotated[
|
||||||
|
Path | None,
|
||||||
|
typer.Option("--pose-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
] = None,
|
||||||
|
split_archive: Annotated[
|
||||||
|
Path | None,
|
||||||
|
typer.Option("--split-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||||
|
] = None,
|
||||||
|
preview_points: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--preview-points", min=1, max=20_000),
|
||||||
|
] = 20_000,
|
||||||
|
) -> None:
|
||||||
|
"""Verify the complete official RELLIS release on worker D."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
manifest = admit_rellis_release(
|
||||||
|
dataset_root,
|
||||||
|
scan_archive=scan_archive,
|
||||||
|
label_archive=label_archive,
|
||||||
|
pose_archive=pose_archive,
|
||||||
|
split_archive=split_archive,
|
||||||
|
preview_points=preview_points,
|
||||||
|
)
|
||||||
|
except RellisAdmissionError as exc:
|
||||||
|
typer.echo(str(exc), err=True)
|
||||||
|
raise typer.Exit(code=2) from exc
|
||||||
|
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
|
@app.command("qualify-rellis-ground")
|
||||||
|
def qualify_rellis_ground_command(
|
||||||
|
dataset_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
runs_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option("--runs-root", file_okay=False, resolve_path=True),
|
||||||
|
],
|
||||||
|
mission_core_commit: Annotated[
|
||||||
|
str,
|
||||||
|
typer.Option("--mission-core-commit", min=7, max=64),
|
||||||
|
],
|
||||||
|
workers: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--workers", min=1, max=32),
|
||||||
|
] = 8,
|
||||||
|
calibration_frames: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--calibration-frames", min=5, max=256),
|
||||||
|
] = 64,
|
||||||
|
preview_points: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--preview-points", min=1, max=20_000),
|
||||||
|
] = 12_000,
|
||||||
|
) -> None:
|
||||||
|
"""Calibrate on RELLIS train and qualify Current/Patchwork++ on validation."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
report = qualify_rellis_ground(
|
||||||
|
dataset_root,
|
||||||
|
runs_root,
|
||||||
|
mission_core_commit=mission_core_commit,
|
||||||
|
parallel_workers=workers,
|
||||||
|
calibration_frame_count=calibration_frames,
|
||||||
|
preview_points=preview_points,
|
||||||
|
)
|
||||||
|
except (RellisAdmissionError, ValueError) as exc:
|
||||||
|
typer.echo(str(exc), err=True)
|
||||||
|
raise typer.Exit(code=2) from exc
|
||||||
|
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app()
|
app()
|
||||||
|
|
|
||||||
|
|
@ -178,6 +178,7 @@ def dataset_gateway_catalog(
|
||||||
dataset_root: Path | None = None,
|
dataset_root: Path | None = None,
|
||||||
admission_manifest_path: Path | None = None,
|
admission_manifest_path: Path | None = None,
|
||||||
rellis_preview_path: Path | None = None,
|
rellis_preview_path: Path | None = None,
|
||||||
|
rellis_admission_path: Path | None = None,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||||
|
|
||||||
|
|
@ -205,6 +206,34 @@ def dataset_gateway_catalog(
|
||||||
rellis_preview = candidate
|
rellis_preview = candidate
|
||||||
except DatasetAdmissionError:
|
except DatasetAdmissionError:
|
||||||
pass
|
pass
|
||||||
|
rellis_admission: dict[str, Any] | None = None
|
||||||
|
if rellis_admission_path is not None and rellis_admission_path.is_file():
|
||||||
|
try:
|
||||||
|
candidate = _bounded_json_object(
|
||||||
|
rellis_admission_path,
|
||||||
|
MAX_STATE_BYTES,
|
||||||
|
"RELLIS admission manifest",
|
||||||
|
)
|
||||||
|
storage = _object(candidate.get("storage"), "RELLIS storage")
|
||||||
|
alignment = _object(candidate.get("alignment"), "RELLIS alignment")
|
||||||
|
release = _object(candidate.get("release"), "RELLIS release")
|
||||||
|
if (
|
||||||
|
candidate.get("schema_version") != "missioncore.rellis-admission/v1"
|
||||||
|
or candidate.get("source_id") != "rellis-3d/v1.1"
|
||||||
|
or candidate.get("status") != "dataset-ready"
|
||||||
|
or storage.get("policy") != "worker-d-only"
|
||||||
|
or storage.get("admitted") is not True
|
||||||
|
or storage.get("path_exposed") is not False
|
||||||
|
or alignment.get("annotated_scan_count") != 13_556
|
||||||
|
or alignment.get("point_label_alignment") != "complete"
|
||||||
|
or alignment.get("pose_index_coverage") != "complete"
|
||||||
|
or not isinstance(release.get("identity_sha256"), str)
|
||||||
|
or len(release["identity_sha256"]) != 64
|
||||||
|
):
|
||||||
|
raise DatasetAdmissionError("RELLIS admission manifest is incompatible")
|
||||||
|
rellis_admission = candidate
|
||||||
|
except DatasetAdmissionError:
|
||||||
|
pass
|
||||||
source_status = (
|
source_status = (
|
||||||
str(admission["status"])
|
str(admission["status"])
|
||||||
if admission is not None
|
if admission is not None
|
||||||
|
|
@ -222,14 +251,18 @@ def dataset_gateway_catalog(
|
||||||
else "configure-dataset-root-on-worker-d"
|
else "configure-dataset-root-on-worker-d"
|
||||||
)
|
)
|
||||||
rellis_status = (
|
rellis_status = (
|
||||||
"smoke-ready"
|
"dataset-ready"
|
||||||
|
if rellis_admission is not None and rellis_preview is not None
|
||||||
|
else "smoke-ready"
|
||||||
if rellis_preview is not None
|
if rellis_preview is not None
|
||||||
else "ready-for-smoke"
|
else "ready-for-smoke"
|
||||||
if storage_admitted
|
if storage_admitted
|
||||||
else "blocked-storage-policy"
|
else "blocked-storage-policy"
|
||||||
)
|
)
|
||||||
catalog_next_action = (
|
catalog_next_action = (
|
||||||
"admit-rellis-ouster-semantickitti-to-worker-d"
|
"qualify-rellis-current-vs-patchworkpp"
|
||||||
|
if rellis_admission is not None
|
||||||
|
else "admit-rellis-ouster-semantickitti-to-worker-d"
|
||||||
if rellis_preview is not None and source_status == "frame-ready"
|
if rellis_preview is not None and source_status == "frame-ready"
|
||||||
else "build-rellis-official-example-smoke"
|
else "build-rellis-official-example-smoke"
|
||||||
if source_status == "frame-ready"
|
if source_status == "frame-ready"
|
||||||
|
|
@ -342,7 +375,9 @@ def dataset_gateway_catalog(
|
||||||
"admission": {
|
"admission": {
|
||||||
"status": rellis_status,
|
"status": rellis_status,
|
||||||
"native_scan": (
|
"native_scan": (
|
||||||
"official-example-compatible"
|
"complete-release-split-aligned"
|
||||||
|
if rellis_admission is not None
|
||||||
|
else "official-example-compatible"
|
||||||
if rellis_preview is not None
|
if rellis_preview is not None
|
||||||
else "requires-official-example-smoke"
|
else "requires-official-example-smoke"
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,417 @@
|
||||||
|
"""Fail-closed admission of the complete official RELLIS-3D release.
|
||||||
|
|
||||||
|
All four source archives remain on the Simulation Worker D drive. Admission
|
||||||
|
indexes the ZIP central directories, proves split/scan/label/pose alignment and
|
||||||
|
publishes only a path-free manifest plus one bounded validation-frame preview.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Any, Final
|
||||||
|
|
||||||
|
from k1link.datasets.gateway import DatasetFrameError, decode_semantic_kitti_frame
|
||||||
|
from k1link.datasets.rellis_smoke import (
|
||||||
|
RELLIS_LABEL_CONFIG_SHA256,
|
||||||
|
RELLIS_LICENSE,
|
||||||
|
RELLIS_REPOSITORY_COMMIT,
|
||||||
|
RELLIS_REPOSITORY_URL,
|
||||||
|
RELLIS_SOURCE_ID,
|
||||||
|
RellisSmokeError,
|
||||||
|
rellis_native_scan_preview,
|
||||||
|
)
|
||||||
|
|
||||||
|
RELLIS_ADMISSION_SCHEMA: Final = "missioncore.rellis-admission/v1"
|
||||||
|
RELLIS_RELEASE_ID: Final = "v1.1"
|
||||||
|
RELLIS_ARCHIVE_ROOT: Final = "rellis-3d/v1.1/archives"
|
||||||
|
RELLIS_SCAN_ARCHIVE: Final = "Rellis_3D_os1_cloud_node_kitti_bin.zip"
|
||||||
|
RELLIS_LABEL_ARCHIVE: Final = "Rellis_3D_os1_labels_20210614.zip"
|
||||||
|
RELLIS_POSE_ARCHIVE: Final = "Rellis_3D_lidar_poses_20210614.zip"
|
||||||
|
RELLIS_SPLIT_ARCHIVE: Final = "Rellis_3D_lidar_split.zip"
|
||||||
|
RELLIS_ARCHIVE_URLS: Final[dict[str, str]] = {
|
||||||
|
RELLIS_SCAN_ARCHIVE: (
|
||||||
|
"https://drive.google.com/uc?id=1lDSVRf_kZrD0zHHMsKJ0V1GN9QATR4wH"
|
||||||
|
),
|
||||||
|
RELLIS_LABEL_ARCHIVE: (
|
||||||
|
"https://drive.google.com/uc?id=12bsblHXtob60KrjV7lGXUQTdC5PhV8Er"
|
||||||
|
),
|
||||||
|
RELLIS_POSE_ARCHIVE: (
|
||||||
|
"https://drive.google.com/uc?id=1V3PT_NJhA41N7TBLp5AbW31d0ztQDQOX"
|
||||||
|
),
|
||||||
|
RELLIS_SPLIT_ARCHIVE: (
|
||||||
|
"https://drive.google.com/uc?id=1raQJPySyqDaHpc53KPnJVl3Bln6HlcVS"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
RELLIS_OBSERVED_SHA256: Final[dict[str, str]] = {
|
||||||
|
RELLIS_LABEL_ARCHIVE: (
|
||||||
|
"03297c30b9f2182c74be93f8564d6ce1349237fcc4f89fc96479fe71bb40d905"
|
||||||
|
),
|
||||||
|
RELLIS_POSE_ARCHIVE: (
|
||||||
|
"6deaf38a9cac3a480cfdb70fa9a1d9278f25c36f5efd2487a3c4a6a400250f67"
|
||||||
|
),
|
||||||
|
RELLIS_SPLIT_ARCHIVE: (
|
||||||
|
"de639728e0058d7d477188028d9b811313a601884ef00d267ce13736be7cf54f"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
RELLIS_SPLIT_COUNTS: Final[dict[str, int]] = {
|
||||||
|
"train": 7_800,
|
||||||
|
"validation": 2_413,
|
||||||
|
"test": 3_343,
|
||||||
|
}
|
||||||
|
RELLIS_SEQUENCE_IDS: Final = ("00000", "00001", "00002", "00003", "00004")
|
||||||
|
RELLIS_ANNOTATED_SCAN_COUNT: Final = 13_556
|
||||||
|
MAX_ARCHIVE_ENTRIES: Final = 40_000
|
||||||
|
MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3
|
||||||
|
MAX_METADATA_MEMBER_BYTES: Final = 32 * 1024**2
|
||||||
|
MAX_PREVIEW_POINTS: Final = 20_000
|
||||||
|
_FRAME_PATH = re.compile(
|
||||||
|
r"^(?P<sequence>0000[0-4])/os1_cloud_node_kitti_bin/"
|
||||||
|
r"(?P<frame>[0-9]{6})\.bin$"
|
||||||
|
)
|
||||||
|
_LABEL_PATH = re.compile(
|
||||||
|
r"^(?P<sequence>0000[0-4])/os1_cloud_node_semantickitti_label_id/"
|
||||||
|
r"(?P<frame>[0-9]{6})\.label$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RellisAdmissionError(RuntimeError):
|
||||||
|
"""The official RELLIS release cannot satisfy its pinned admission contract."""
|
||||||
|
|
||||||
|
|
||||||
|
def admit_rellis_release(
|
||||||
|
dataset_root: Path,
|
||||||
|
*,
|
||||||
|
scan_archive: Path | None = None,
|
||||||
|
label_archive: Path | None = None,
|
||||||
|
pose_archive: Path | None = None,
|
||||||
|
split_archive: Path | None = None,
|
||||||
|
preview_points: int = MAX_PREVIEW_POINTS,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Verify the full official release without extracting its source archives."""
|
||||||
|
|
||||||
|
root = dataset_root.expanduser().absolute()
|
||||||
|
if not _is_worker_dataset_root(root):
|
||||||
|
raise RellisAdmissionError("RELLIS admission requires the canonical worker D root")
|
||||||
|
if not 1 <= preview_points <= MAX_PREVIEW_POINTS:
|
||||||
|
raise RellisAdmissionError("RELLIS preview point limit is outside the admitted range")
|
||||||
|
archive_root = root / RELLIS_ARCHIVE_ROOT
|
||||||
|
paths = {
|
||||||
|
RELLIS_SCAN_ARCHIVE: _archive_path(archive_root, scan_archive, RELLIS_SCAN_ARCHIVE),
|
||||||
|
RELLIS_LABEL_ARCHIVE: _archive_path(archive_root, label_archive, RELLIS_LABEL_ARCHIVE),
|
||||||
|
RELLIS_POSE_ARCHIVE: _archive_path(archive_root, pose_archive, RELLIS_POSE_ARCHIVE),
|
||||||
|
RELLIS_SPLIT_ARCHIVE: _archive_path(archive_root, split_archive, RELLIS_SPLIT_ARCHIVE),
|
||||||
|
}
|
||||||
|
if any(not path.is_file() for path in paths.values()):
|
||||||
|
raise RellisAdmissionError("one or more official RELLIS archives are unavailable")
|
||||||
|
|
||||||
|
descriptors: dict[str, dict[str, Any]] = {}
|
||||||
|
for filename, path in paths.items():
|
||||||
|
digest = _sha256_file(path)
|
||||||
|
observed = RELLIS_OBSERVED_SHA256.get(filename)
|
||||||
|
if observed is not None and digest != observed:
|
||||||
|
raise RellisAdmissionError(f"{filename} differs from the pinned observed release")
|
||||||
|
descriptors[filename] = {
|
||||||
|
"filename": filename,
|
||||||
|
"source_url": RELLIS_ARCHIVE_URLS[filename],
|
||||||
|
"size_bytes": path.stat().st_size,
|
||||||
|
"sha256": digest,
|
||||||
|
"vendor_checksum_available": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with (
|
||||||
|
zipfile.ZipFile(paths[RELLIS_SCAN_ARCHIVE]) as scans_zip,
|
||||||
|
zipfile.ZipFile(paths[RELLIS_LABEL_ARCHIVE]) as labels_zip,
|
||||||
|
zipfile.ZipFile(paths[RELLIS_POSE_ARCHIVE]) as poses_zip,
|
||||||
|
zipfile.ZipFile(paths[RELLIS_SPLIT_ARCHIVE]) as splits_zip,
|
||||||
|
):
|
||||||
|
scan_members = _member_index(scans_zip)
|
||||||
|
label_members = _member_index(labels_zip)
|
||||||
|
pose_members = _member_index(poses_zip)
|
||||||
|
split_members = _member_index(splits_zip)
|
||||||
|
splits = _read_splits(splits_zip, split_members)
|
||||||
|
split_pairs = {
|
||||||
|
pair for split in splits.values() for pair in split
|
||||||
|
}
|
||||||
|
if len(split_pairs) != RELLIS_ANNOTATED_SCAN_COUNT:
|
||||||
|
raise RellisAdmissionError("RELLIS split union is incomplete or overlapping")
|
||||||
|
|
||||||
|
indexed_scans = {
|
||||||
|
path: member
|
||||||
|
for path, member in scan_members.items()
|
||||||
|
if _FRAME_PATH.fullmatch(path)
|
||||||
|
}
|
||||||
|
indexed_labels = {
|
||||||
|
path: member
|
||||||
|
for path, member in label_members.items()
|
||||||
|
if _LABEL_PATH.fullmatch(path)
|
||||||
|
}
|
||||||
|
expected_scans = {point_path for point_path, _ in split_pairs}
|
||||||
|
expected_labels = {label_path for _, label_path in split_pairs}
|
||||||
|
if set(indexed_scans) != expected_scans:
|
||||||
|
raise RellisAdmissionError("RELLIS scan archive does not match the official splits")
|
||||||
|
if set(indexed_labels) != expected_labels:
|
||||||
|
raise RellisAdmissionError(
|
||||||
|
"RELLIS label archive does not match the official splits"
|
||||||
|
)
|
||||||
|
_validate_member_alignment(indexed_scans, indexed_labels, split_pairs)
|
||||||
|
pose_counts = _validate_poses(poses_zip, pose_members, split_pairs)
|
||||||
|
|
||||||
|
validation_point, validation_label = splits["validation"][0]
|
||||||
|
point_bytes = scans_zip.read(indexed_scans[validation_point])
|
||||||
|
label_bytes = labels_zip.read(indexed_labels[validation_label])
|
||||||
|
frame = decode_semantic_kitti_frame(point_bytes, label_bytes)
|
||||||
|
frame_id = _qualified_frame_id(validation_point)
|
||||||
|
preview = rellis_native_scan_preview(
|
||||||
|
frame,
|
||||||
|
frame_id=frame_id,
|
||||||
|
maximum_points=preview_points,
|
||||||
|
source_evidence={
|
||||||
|
"repository_url": RELLIS_REPOSITORY_URL,
|
||||||
|
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||||
|
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
|
||||||
|
"point_sha256": hashlib.sha256(point_bytes).hexdigest(),
|
||||||
|
"label_sha256": hashlib.sha256(label_bytes).hexdigest(),
|
||||||
|
"license": RELLIS_LICENSE,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
except (OSError, KeyError, zipfile.BadZipFile, DatasetFrameError, RellisSmokeError) as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS archives could not be verified") from exc
|
||||||
|
|
||||||
|
sequence_counts = Counter(
|
||||||
|
_FRAME_PATH.fullmatch(point_path).group("sequence") # type: ignore[union-attr]
|
||||||
|
for point_path, _ in split_pairs
|
||||||
|
)
|
||||||
|
install_identity = hashlib.sha256(
|
||||||
|
json.dumps(descriptors, sort_keys=True, separators=(",", ":")).encode()
|
||||||
|
).hexdigest()
|
||||||
|
install_root = root / "rellis-3d/v1.1/installs" / install_identity
|
||||||
|
preview_path = install_root / "previews" / f"{frame_id}.json"
|
||||||
|
_atomic_json(preview_path, preview)
|
||||||
|
manifest = {
|
||||||
|
"schema_version": RELLIS_ADMISSION_SCHEMA,
|
||||||
|
"source_id": RELLIS_SOURCE_ID,
|
||||||
|
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||||
|
"status": "dataset-ready",
|
||||||
|
"storage": {
|
||||||
|
"policy": "worker-d-only",
|
||||||
|
"admitted": True,
|
||||||
|
"canonical_root": True,
|
||||||
|
"path_exposed": False,
|
||||||
|
},
|
||||||
|
"release": {
|
||||||
|
"repository_url": RELLIS_REPOSITORY_URL,
|
||||||
|
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||||
|
"license": RELLIS_LICENSE,
|
||||||
|
"archives": descriptors,
|
||||||
|
"identity_sha256": install_identity,
|
||||||
|
},
|
||||||
|
"alignment": {
|
||||||
|
"annotated_scan_count": len(split_pairs),
|
||||||
|
"split_counts": RELLIS_SPLIT_COUNTS,
|
||||||
|
"sequence_counts": dict(sorted(sequence_counts.items())),
|
||||||
|
"pose_counts": pose_counts,
|
||||||
|
"point_label_alignment": "complete",
|
||||||
|
"pose_index_coverage": "complete",
|
||||||
|
},
|
||||||
|
"frame": {
|
||||||
|
"frame_id": frame_id,
|
||||||
|
"split": "validation",
|
||||||
|
"source_point_count": frame.point_count,
|
||||||
|
"preview_point_count": int(preview["point_count"]),
|
||||||
|
"preview_sha256": _sha256_file(preview_path),
|
||||||
|
},
|
||||||
|
"coordinate_frame": {
|
||||||
|
"frame_id": "sensor/lidar/os1",
|
||||||
|
"x": "forward",
|
||||||
|
"y": "left",
|
||||||
|
"z": "up",
|
||||||
|
"transform_applied": False,
|
||||||
|
},
|
||||||
|
"next_action": "calibrate-on-train-then-qualify-validation",
|
||||||
|
}
|
||||||
|
_atomic_json(root / "state/rellis-3d-v1.1.json", manifest)
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def read_rellis_splits(dataset_root: Path) -> dict[str, tuple[tuple[str, str], ...]]:
|
||||||
|
"""Read the already admitted immutable split index."""
|
||||||
|
|
||||||
|
root = dataset_root.expanduser().absolute()
|
||||||
|
if not _is_worker_dataset_root(root):
|
||||||
|
raise RellisAdmissionError("RELLIS split access requires the canonical worker D root")
|
||||||
|
split_path = root / RELLIS_ARCHIVE_ROOT / RELLIS_SPLIT_ARCHIVE
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(split_path) as source:
|
||||||
|
return _read_splits(source, _member_index(source))
|
||||||
|
except (OSError, zipfile.BadZipFile) as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS split archive cannot be indexed") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_path(root: Path, value: Path | None, filename: str) -> Path:
|
||||||
|
return value.expanduser().absolute() if value is not None else root / filename
|
||||||
|
|
||||||
|
|
||||||
|
def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]:
|
||||||
|
members = source.infolist()
|
||||||
|
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
|
||||||
|
raise RellisAdmissionError("RELLIS archive entry count is outside the admitted range")
|
||||||
|
total = 0
|
||||||
|
result: dict[str, zipfile.ZipInfo] = {}
|
||||||
|
for member in members:
|
||||||
|
path = PurePosixPath(member.filename.replace("\\", "/"))
|
||||||
|
if path.is_absolute() or ".." in path.parts:
|
||||||
|
raise RellisAdmissionError("RELLIS archive contains an unsafe member path")
|
||||||
|
if (member.external_attr >> 16) & 0o170000 == 0o120000:
|
||||||
|
raise RellisAdmissionError("RELLIS archive contains a symbolic link")
|
||||||
|
total += member.file_size
|
||||||
|
if total > MAX_UNCOMPRESSED_BYTES:
|
||||||
|
raise RellisAdmissionError("RELLIS archive exceeds the uncompressed safety limit")
|
||||||
|
if member.is_dir():
|
||||||
|
continue
|
||||||
|
normalized = "/".join(part for part in path.parts if part not in {".", "Rellis-3D"})
|
||||||
|
if not normalized or normalized in result:
|
||||||
|
raise RellisAdmissionError("RELLIS archive member identity is ambiguous")
|
||||||
|
result[normalized] = member
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _read_splits(
|
||||||
|
source: zipfile.ZipFile,
|
||||||
|
members: dict[str, zipfile.ZipInfo],
|
||||||
|
) -> dict[str, tuple[tuple[str, str], ...]]:
|
||||||
|
result: dict[str, tuple[tuple[str, str], ...]] = {}
|
||||||
|
for split, expected in RELLIS_SPLIT_COUNTS.items():
|
||||||
|
filename = "pt_val.lst" if split == "validation" else f"pt_{split}.lst"
|
||||||
|
matches = [member for path, member in members.items() if path.endswith(filename)]
|
||||||
|
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
|
||||||
|
raise RellisAdmissionError(f"RELLIS {split} split index is unavailable")
|
||||||
|
try:
|
||||||
|
lines = source.read(matches[0]).decode("utf-8").splitlines()
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS split index is not UTF-8") from exc
|
||||||
|
pairs: list[tuple[str, str]] = []
|
||||||
|
for raw in lines:
|
||||||
|
parts = raw.strip().replace("\\", "/").split()
|
||||||
|
if len(parts) != 2:
|
||||||
|
raise RellisAdmissionError("RELLIS split row is malformed")
|
||||||
|
point_path, label_path = parts
|
||||||
|
point_match = _FRAME_PATH.fullmatch(point_path)
|
||||||
|
label_match = _LABEL_PATH.fullmatch(label_path)
|
||||||
|
if (
|
||||||
|
point_match is None
|
||||||
|
or label_match is None
|
||||||
|
or point_match.group("sequence") != label_match.group("sequence")
|
||||||
|
or point_match.group("frame") != label_match.group("frame")
|
||||||
|
):
|
||||||
|
raise RellisAdmissionError("RELLIS split point and label identities differ")
|
||||||
|
pairs.append((point_path, label_path))
|
||||||
|
if len(pairs) != expected or len(set(pairs)) != expected:
|
||||||
|
raise RellisAdmissionError(f"RELLIS {split} split count differs")
|
||||||
|
result[split] = tuple(pairs)
|
||||||
|
if sum(map(len, result.values())) != len(
|
||||||
|
{pair for values in result.values() for pair in values}
|
||||||
|
):
|
||||||
|
raise RellisAdmissionError("RELLIS split indexes overlap")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_member_alignment(
|
||||||
|
scans: dict[str, zipfile.ZipInfo],
|
||||||
|
labels: dict[str, zipfile.ZipInfo],
|
||||||
|
pairs: set[tuple[str, str]],
|
||||||
|
) -> None:
|
||||||
|
for point_path, label_path in pairs:
|
||||||
|
point = scans[point_path]
|
||||||
|
label = labels[label_path]
|
||||||
|
if (
|
||||||
|
point.file_size <= 0
|
||||||
|
or point.file_size % 16
|
||||||
|
or label.file_size % 4
|
||||||
|
or point.file_size // 16 != label.file_size // 4
|
||||||
|
):
|
||||||
|
raise RellisAdmissionError("RELLIS point and label member sizes are not aligned")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_poses(
|
||||||
|
source: zipfile.ZipFile,
|
||||||
|
members: dict[str, zipfile.ZipInfo],
|
||||||
|
pairs: set[tuple[str, str]],
|
||||||
|
) -> dict[str, int]:
|
||||||
|
result: dict[str, int] = {}
|
||||||
|
required_by_sequence: dict[str, set[int]] = {
|
||||||
|
sequence: set() for sequence in RELLIS_SEQUENCE_IDS
|
||||||
|
}
|
||||||
|
for point_path, _ in pairs:
|
||||||
|
match = _FRAME_PATH.fullmatch(point_path)
|
||||||
|
assert match is not None
|
||||||
|
required_by_sequence[match.group("sequence")].add(int(match.group("frame")))
|
||||||
|
for sequence in RELLIS_SEQUENCE_IDS:
|
||||||
|
path = f"{sequence}/poses.txt"
|
||||||
|
matches = [member for name, member in members.items() if name.endswith(path)]
|
||||||
|
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
|
||||||
|
raise RellisAdmissionError(f"RELLIS poses are unavailable for sequence {sequence}")
|
||||||
|
try:
|
||||||
|
rows = source.read(matches[0]).decode("utf-8").splitlines()
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS pose file is not UTF-8") from exc
|
||||||
|
for row in rows:
|
||||||
|
values = row.split()
|
||||||
|
if len(values) != 12:
|
||||||
|
raise RellisAdmissionError("RELLIS pose row is not a 3x4 transform")
|
||||||
|
try:
|
||||||
|
tuple(float(value) for value in values)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS pose row is not numeric") from exc
|
||||||
|
if not rows or max(required_by_sequence[sequence], default=-1) >= len(rows):
|
||||||
|
raise RellisAdmissionError("RELLIS pose index does not cover every split frame")
|
||||||
|
result[sequence] = len(rows)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _qualified_frame_id(point_path: str) -> str:
|
||||||
|
match = _FRAME_PATH.fullmatch(point_path)
|
||||||
|
if match is None:
|
||||||
|
raise RellisAdmissionError("RELLIS frame path is incompatible")
|
||||||
|
return f"{match.group('sequence')}-{match.group('frame')}"
|
||||||
|
|
||||||
|
|
||||||
|
def _is_worker_dataset_root(root: Path) -> bool:
|
||||||
|
return str(root).replace("\\", "/").rstrip("/").lower() == (
|
||||||
|
"/mnt/d/ndc_missioncore/datasets"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
try:
|
||||||
|
with path.open("rb") as source:
|
||||||
|
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
except OSError as exc:
|
||||||
|
raise RellisAdmissionError("RELLIS artifact cannot be hashed") from exc
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tempfile.NamedTemporaryFile(
|
||||||
|
mode="w",
|
||||||
|
dir=path.parent,
|
||||||
|
encoding="utf-8",
|
||||||
|
delete=False,
|
||||||
|
) as temporary:
|
||||||
|
temporary_path = Path(temporary.name)
|
||||||
|
json.dump(document, temporary, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||||
|
temporary.write("\n")
|
||||||
|
temporary.flush()
|
||||||
|
os.fsync(temporary.fileno())
|
||||||
|
os.replace(temporary_path, path)
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
"""Train-calibrated, algorithm-scoped RELLIS OS1 Patchwork++ profile."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Final
|
||||||
|
|
||||||
|
RELLIS_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.rellis-patchwork-profile/v1"
|
||||||
|
RELLIS_URDF_URL: Final = (
|
||||||
|
"https://raw.githubusercontent.com/unmannedlab/RELLIS-3D/"
|
||||||
|
"c17a118fcaed1559f03cc32cc3a91dedc557f8b8/"
|
||||||
|
"catkin_ws/src/platform_description/urdf/warthog.urdf.xacro"
|
||||||
|
)
|
||||||
|
RELLIS_URDF_SHA256: Final = (
|
||||||
|
"bcb66a05b8818f06479bb4a17ce93e9d66cb8fdaab726c437596807288116ac3"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RellisPatchworkProfile:
|
||||||
|
"""Frozen provider profile derived without looking at validation labels."""
|
||||||
|
|
||||||
|
patchwork_sensor_height_proxy_m: float
|
||||||
|
calibration_identity_sha256: str
|
||||||
|
calibration_frame_count: int
|
||||||
|
calibration_median_absolute_deviation_m: float
|
||||||
|
patchwork_minimum_range_m: float = 2.7
|
||||||
|
patchwork_maximum_range_m: float = 80.0
|
||||||
|
profile_id: str = "rellis-warthog-os1-patchwork/v1"
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
values = (
|
||||||
|
self.patchwork_sensor_height_proxy_m,
|
||||||
|
self.calibration_median_absolute_deviation_m,
|
||||||
|
self.patchwork_minimum_range_m,
|
||||||
|
self.patchwork_maximum_range_m,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not self.profile_id
|
||||||
|
or not all(math.isfinite(value) for value in values)
|
||||||
|
or not 0.5 <= self.patchwork_sensor_height_proxy_m <= 3.0
|
||||||
|
or not 5 <= self.calibration_frame_count <= 256
|
||||||
|
or self.calibration_median_absolute_deviation_m < 0
|
||||||
|
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
|
||||||
|
or len(self.calibration_identity_sha256) != 64
|
||||||
|
or any(
|
||||||
|
character not in "0123456789abcdef"
|
||||||
|
for character in self.calibration_identity_sha256
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ValueError("RELLIS Patchwork++ profile is invalid")
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema_version": RELLIS_PATCHWORK_PROFILE_SCHEMA,
|
||||||
|
"profile_id": self.profile_id,
|
||||||
|
"source_id": "rellis-3d/v1.1",
|
||||||
|
"representation": "native-scan",
|
||||||
|
"sensor_frame": {
|
||||||
|
"frame_id": "sensor/lidar/os1",
|
||||||
|
"handedness": "right",
|
||||||
|
"x": "forward",
|
||||||
|
"y": "left",
|
||||||
|
"z": "up",
|
||||||
|
"one_revolution": True,
|
||||||
|
},
|
||||||
|
"height": {
|
||||||
|
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
|
||||||
|
"evidence": "train-split-ground-label-calibration",
|
||||||
|
"calibration_frame_count": self.calibration_frame_count,
|
||||||
|
"median_absolute_deviation_m": (
|
||||||
|
self.calibration_median_absolute_deviation_m
|
||||||
|
),
|
||||||
|
"calibration_identity_sha256": self.calibration_identity_sha256,
|
||||||
|
"validation_labels_used": False,
|
||||||
|
},
|
||||||
|
"patchworkpp": {
|
||||||
|
"provider_id": "patchworkpp/v1.4.1",
|
||||||
|
"minimum_range_m": self.patchwork_minimum_range_m,
|
||||||
|
"maximum_range_m": self.patchwork_maximum_range_m,
|
||||||
|
"enable_rnr": True,
|
||||||
|
"enable_rvpf": True,
|
||||||
|
"enable_tgr": True,
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"vehicle_urdf": {
|
||||||
|
"url": RELLIS_URDF_URL,
|
||||||
|
"sha256": RELLIS_URDF_SHA256,
|
||||||
|
"use": "topology-cross-check-only",
|
||||||
|
"exact_physical_height_claimed": False,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scope": {
|
||||||
|
"patchworkpp_eligible": True,
|
||||||
|
"normalized_scan_produced": False,
|
||||||
|
"complete_vehicle_transform_known": False,
|
||||||
|
"deskew_claimed": False,
|
||||||
|
},
|
||||||
|
"authority": {
|
||||||
|
"qualification_only": True,
|
||||||
|
"navigation_or_safety_accepted": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -429,6 +429,9 @@ app.include_router(
|
||||||
dataset_rellis_preview_provider=lambda: (
|
dataset_rellis_preview_provider=lambda: (
|
||||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
||||||
),
|
),
|
||||||
|
dataset_rellis_admission_provider=lambda: (
|
||||||
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-admission.json"
|
||||||
|
),
|
||||||
dataset_ground_preview_provider=lambda: (
|
dataset_ground_preview_provider=lambda: (
|
||||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -64,6 +64,7 @@ def build_lidar_router(
|
||||||
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
||||||
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
||||||
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
|
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
|
||||||
|
dataset_rellis_admission_provider: DatasetArtifactProvider = lambda: None,
|
||||||
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
||||||
) -> APIRouter:
|
) -> APIRouter:
|
||||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||||
|
|
@ -73,6 +74,7 @@ def build_lidar_router(
|
||||||
return dataset_gateway_catalog(
|
return dataset_gateway_catalog(
|
||||||
admission_manifest_path=dataset_admission_provider(),
|
admission_manifest_path=dataset_admission_provider(),
|
||||||
rellis_preview_path=dataset_rellis_preview_provider(),
|
rellis_preview_path=dataset_rellis_preview_provider(),
|
||||||
|
rellis_admission_path=dataset_rellis_admission_provider(),
|
||||||
)
|
)
|
||||||
|
|
||||||
@router.get("/dataset-gateway/preview")
|
@router.get("/dataset-gateway/preview")
|
||||||
|
|
|
||||||
|
|
@ -41,17 +41,21 @@ DETAIL_SCHEMA: Final = "missioncore.polygon-run-detail/v1"
|
||||||
GROUND_QUALIFICATION_SCHEMA: Final = "missioncore.polygon-ground-qualification/v1"
|
GROUND_QUALIFICATION_SCHEMA: Final = "missioncore.polygon-ground-qualification/v1"
|
||||||
GROUND_FAILURE_PREVIEW_SCHEMA: Final = "missioncore.polygon-ground-failure-preview/v1"
|
GROUND_FAILURE_PREVIEW_SCHEMA: Final = "missioncore.polygon-ground-failure-preview/v1"
|
||||||
GROUND_REPORT_ARTIFACT_KIND: Final = "goose-ground-qualification-report"
|
GROUND_REPORT_ARTIFACT_KIND: Final = "goose-ground-qualification-report"
|
||||||
|
RELLIS_REPORT_ARTIFACT_KIND: Final = "rellis-ground-qualification-report"
|
||||||
GROUND_FAILURE_ARTIFACT_KIND: Final = "goose-ground-qualification-failure-preview"
|
GROUND_FAILURE_ARTIFACT_KIND: Final = "goose-ground-qualification-failure-preview"
|
||||||
GROUND_REPORT_SCHEMA: Final = "missioncore.goose-ground-qualification-report/v1"
|
GROUND_REPORT_SCHEMA: Final = "missioncore.goose-ground-qualification-report/v1"
|
||||||
|
RELLIS_REPORT_SCHEMA: Final = "missioncore.rellis-ground-qualification-report/v1"
|
||||||
GROUND_FAILURE_SCHEMA: Final = "missioncore.goose-ground-qualification-failure-preview/v1"
|
GROUND_FAILURE_SCHEMA: Final = "missioncore.goose-ground-qualification-failure-preview/v1"
|
||||||
GROUND_REVIEW_SCHEMA: Final = "missioncore.polygon-ground-review/v2"
|
GROUND_REVIEW_SCHEMA: Final = "missioncore.polygon-ground-review/v2"
|
||||||
GROUND_REVIEW_FRAME_SCHEMA: Final = "missioncore.polygon-ground-review-frame/v1"
|
GROUND_REVIEW_FRAME_SCHEMA: Final = "missioncore.polygon-ground-review-frame/v1"
|
||||||
GOOSE_REVIEW_PACK_SCHEMA: Final = "missioncore.goose-ground-review-pack/v1"
|
GOOSE_REVIEW_PACK_SCHEMA: Final = "missioncore.goose-ground-review-pack/v1"
|
||||||
GOOSE_REVIEW_FRAME_SCHEMA: Final = "missioncore.goose-ground-review-frame/v1"
|
GOOSE_REVIEW_FRAME_SCHEMA: Final = "missioncore.goose-ground-review-frame/v1"
|
||||||
|
RELLIS_REVIEW_PACK_SCHEMA: Final = "missioncore.rellis-ground-review-pack/v1"
|
||||||
|
RELLIS_REVIEW_FRAME_SCHEMA: Final = "missioncore.rellis-ground-review-frame/v1"
|
||||||
MAX_QUALIFICATION_ARTIFACT_BYTES: Final = 32 * 1024**2
|
MAX_QUALIFICATION_ARTIFACT_BYTES: Final = 32 * 1024**2
|
||||||
MAX_REVIEW_MANIFEST_BYTES: Final = 8 * 1024**2
|
MAX_REVIEW_MANIFEST_BYTES: Final = 8 * 1024**2
|
||||||
MAX_REVIEW_FRAME_BYTES: Final = 2 * 1024**2
|
MAX_REVIEW_FRAME_BYTES: Final = 2 * 1024**2
|
||||||
MAX_REVIEW_FRAMES: Final = 2_000
|
MAX_REVIEW_FRAMES: Final = 3_000
|
||||||
MAX_REVIEW_POINTS: Final = 20_000
|
MAX_REVIEW_POINTS: Final = 20_000
|
||||||
COMMIT_PATTERN: Final = re.compile(r"^[a-f0-9]{40}$")
|
COMMIT_PATTERN: Final = re.compile(r"^[a-f0-9]{40}$")
|
||||||
GOOSE_FRAME_ID_PATTERN: Final = re.compile(
|
GOOSE_FRAME_ID_PATTERN: Final = re.compile(
|
||||||
|
|
@ -201,8 +205,8 @@ def build_polygon_router(
|
||||||
report = _read_registered_json(
|
report = _read_registered_json(
|
||||||
store,
|
store,
|
||||||
run,
|
run,
|
||||||
artifact_kind=GROUND_REPORT_ARTIFACT_KIND,
|
artifact_kind=(GROUND_REPORT_ARTIFACT_KIND, RELLIS_REPORT_ARTIFACT_KIND),
|
||||||
expected_schema=GROUND_REPORT_SCHEMA,
|
expected_schema=(GROUND_REPORT_SCHEMA, RELLIS_REPORT_SCHEMA),
|
||||||
)
|
)
|
||||||
required = (
|
required = (
|
||||||
"identity_sha256",
|
"identity_sha256",
|
||||||
|
|
@ -305,7 +309,7 @@ def build_polygon_router(
|
||||||
"ground_iou_delta",
|
"ground_iou_delta",
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
**_goose_frame_identity(frame["frame_id"]),
|
**_dataset_frame_identity(frame),
|
||||||
}
|
}
|
||||||
for frame in manifest["frames"]
|
for frame in manifest["frames"]
|
||||||
],
|
],
|
||||||
|
|
@ -348,7 +352,11 @@ def build_polygon_router(
|
||||||
) from exc
|
) from exc
|
||||||
point_count = int(frame["point_count"])
|
point_count = int(frame["point_count"])
|
||||||
if (
|
if (
|
||||||
schema != GOOSE_REVIEW_FRAME_SCHEMA
|
schema
|
||||||
|
not in {
|
||||||
|
GOOSE_REVIEW_FRAME_SCHEMA,
|
||||||
|
RELLIS_REVIEW_FRAME_SCHEMA,
|
||||||
|
}
|
||||||
or stored_frame_id != frame_id
|
or stored_frame_id != frame_id
|
||||||
or source_point_count != frame["source_point_count"]
|
or source_point_count != frame["source_point_count"]
|
||||||
or xyz_cm.dtype != np.dtype("<i2")
|
or xyz_cm.dtype != np.dtype("<i2")
|
||||||
|
|
@ -562,11 +570,17 @@ def _read_registered_json(
|
||||||
store: QualificationRunStore,
|
store: QualificationRunStore,
|
||||||
run: QualificationRun,
|
run: QualificationRun,
|
||||||
*,
|
*,
|
||||||
artifact_kind: str,
|
artifact_kind: str | tuple[str, ...],
|
||||||
expected_schema: str,
|
expected_schema: str | tuple[str, ...],
|
||||||
frame_id: str | None = None,
|
frame_id: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
candidates = [artifact for artifact in run.artifacts if artifact.kind == artifact_kind]
|
artifact_kinds = (artifact_kind,) if isinstance(artifact_kind, str) else artifact_kind
|
||||||
|
expected_schemas = (
|
||||||
|
(expected_schema,) if isinstance(expected_schema, str) else expected_schema
|
||||||
|
)
|
||||||
|
candidates = [
|
||||||
|
artifact for artifact in run.artifacts if artifact.kind in artifact_kinds
|
||||||
|
]
|
||||||
if frame_id is not None:
|
if frame_id is not None:
|
||||||
candidates = [
|
candidates = [
|
||||||
artifact for artifact in candidates if Path(artifact.relative_path).stem == frame_id
|
artifact for artifact in candidates if Path(artifact.relative_path).stem == frame_id
|
||||||
|
|
@ -611,7 +625,7 @@ def _read_registered_json(
|
||||||
status_code=500,
|
status_code=500,
|
||||||
detail="Квалификационный артефакт не является допустимым JSON.",
|
detail="Квалификационный артефакт не является допустимым JSON.",
|
||||||
) from exc
|
) from exc
|
||||||
if not isinstance(value, dict) or value.get("schema_version") != expected_schema:
|
if not isinstance(value, dict) or value.get("schema_version") not in expected_schemas:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=500,
|
status_code=500,
|
||||||
detail="Квалификационный артефакт имеет неизвестную схему.",
|
detail="Квалификационный артефакт имеет неизвестную схему.",
|
||||||
|
|
@ -676,7 +690,8 @@ def _read_ground_review_manifest(
|
||||||
safety = manifest.get("safety") if isinstance(manifest, dict) else None
|
safety = manifest.get("safety") if isinstance(manifest, dict) else None
|
||||||
if (
|
if (
|
||||||
not isinstance(manifest, dict)
|
not isinstance(manifest, dict)
|
||||||
or manifest.get("schema_version") != GOOSE_REVIEW_PACK_SCHEMA
|
or manifest.get("schema_version")
|
||||||
|
not in {GOOSE_REVIEW_PACK_SCHEMA, RELLIS_REVIEW_PACK_SCHEMA}
|
||||||
or manifest.get("source_run_id") != run_id
|
or manifest.get("source_run_id") != run_id
|
||||||
or not isinstance(manifest.get("identity_sha256"), str)
|
or not isinstance(manifest.get("identity_sha256"), str)
|
||||||
or not re.fullmatch(r"[a-f0-9]{64}", manifest["identity_sha256"])
|
or not re.fullmatch(r"[a-f0-9]{64}", manifest["identity_sha256"])
|
||||||
|
|
@ -725,7 +740,27 @@ def _read_ground_review_manifest(
|
||||||
return pack_root, manifest
|
return pack_root, manifest
|
||||||
|
|
||||||
|
|
||||||
def _goose_frame_identity(frame_id: str) -> dict[str, str | int]:
|
def _dataset_frame_identity(frame: dict[str, Any]) -> dict[str, str | int | None]:
|
||||||
|
frame_id = frame["frame_id"]
|
||||||
|
explicit_sequence = frame.get("dataset_sequence_id")
|
||||||
|
explicit_number = frame.get("dataset_frame_number")
|
||||||
|
if explicit_sequence is not None or explicit_number is not None:
|
||||||
|
if (
|
||||||
|
not isinstance(explicit_sequence, str)
|
||||||
|
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,95}", explicit_sequence)
|
||||||
|
is None
|
||||||
|
or not isinstance(explicit_number, int)
|
||||||
|
or explicit_number < 0
|
||||||
|
):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=500,
|
||||||
|
detail="Идентичность dataset-кадра нарушает контракт.",
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"dataset_sequence_id": explicit_sequence,
|
||||||
|
"dataset_frame_number": explicit_number,
|
||||||
|
"sensor_timestamp_ns": None,
|
||||||
|
}
|
||||||
match = GOOSE_FRAME_ID_PATTERN.fullmatch(frame_id)
|
match = GOOSE_FRAME_ID_PATTERN.fullmatch(frame_id)
|
||||||
if match is None:
|
if match is None:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
|
|
|
||||||
|
|
@ -435,6 +435,44 @@ def test_polygon_api_rejects_tampered_review_frame(tmp_path: Path) -> None:
|
||||||
assert "SHA-256" in failure.value.detail or "файл" in failure.value.detail.lower()
|
assert "SHA-256" in failure.value.detail or "файл" in failure.value.detail.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_polygon_api_publishes_rellis_sequence_identity(tmp_path: Path) -> None:
|
||||||
|
root = tmp_path / "runs"
|
||||||
|
run, _ = _qualification_run(root)
|
||||||
|
frame_id = "rellis-00000-000307"
|
||||||
|
frame_path = _review_pack(root, run.run_id, frame_id)
|
||||||
|
with np.load(frame_path, allow_pickle=False) as source:
|
||||||
|
payload = {key: source[key] for key in source.files}
|
||||||
|
payload["schema"] = np.asarray(["missioncore.rellis-ground-review-frame/v1"])
|
||||||
|
np.savez_compressed(frame_path, **payload)
|
||||||
|
manifest_path = frame_path.parent.parent / "manifest.json"
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
manifest["schema_version"] = "missioncore.rellis-ground-review-pack/v1"
|
||||||
|
manifest["source_id"] = "rellis-3d/v1.1"
|
||||||
|
manifest["frames"][0]["dataset_sequence_id"] = "00000"
|
||||||
|
manifest["frames"][0]["dataset_frame_number"] = 307
|
||||||
|
manifest["frames"][0]["sha256"] = hashlib.sha256(frame_path.read_bytes()).hexdigest()
|
||||||
|
manifest["frames"][0]["byte_length"] = frame_path.stat().st_size
|
||||||
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||||
|
router = build_polygon_router(root_provider=lambda: root)
|
||||||
|
|
||||||
|
review = _endpoint(
|
||||||
|
router,
|
||||||
|
"/api/v1/polygon/runs/{run_id}/qualification/review",
|
||||||
|
"GET",
|
||||||
|
)(run_id=run.run_id)
|
||||||
|
frame = _endpoint(
|
||||||
|
router,
|
||||||
|
"/api/v1/polygon/runs/{run_id}/qualification/review/frames/{frame_id}",
|
||||||
|
"GET",
|
||||||
|
)(run_id=run.run_id, frame_id=frame_id)
|
||||||
|
|
||||||
|
assert review["source_id"] == "rellis-3d/v1.1"
|
||||||
|
assert review["frames"][0]["dataset_sequence_id"] == "00000"
|
||||||
|
assert review["frames"][0]["dataset_frame_number"] == 307
|
||||||
|
assert review["frames"][0]["sensor_timestamp_ns"] is None
|
||||||
|
assert frame["frame_id"] == frame_id
|
||||||
|
|
||||||
|
|
||||||
class _FakeWorkerGateway:
|
class _FakeWorkerGateway:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.calls: list[tuple[str, str]] = []
|
self.calls: list[tuple[str, str]] = []
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,78 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.datasets.rellis_admission import RellisAdmissionError, _member_index
|
||||||
|
from k1link.datasets.rellis_profile import RellisPatchworkProfile
|
||||||
|
from k1link.datasets.rellis_qualification import calibrate_rellis_sensor_height
|
||||||
|
|
||||||
|
|
||||||
|
def _frame(height_m: float) -> tuple[bytes, bytes]:
|
||||||
|
point_count = 512
|
||||||
|
points = np.zeros((point_count, 4), dtype="<f4")
|
||||||
|
points[:, 0] = np.linspace(3.0, 12.0, point_count, dtype=np.float32)
|
||||||
|
points[:, 2] = -height_m
|
||||||
|
points[:, 3] = 0.5
|
||||||
|
labels = np.full(point_count, 1, dtype="<u4")
|
||||||
|
return points.tobytes(), labels.tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rellis_height_calibration_uses_only_stratified_train_frames(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
scan_path = tmp_path / "scans.zip"
|
||||||
|
label_path = tmp_path / "labels.zip"
|
||||||
|
pairs: list[tuple[str, str]] = []
|
||||||
|
with (
|
||||||
|
zipfile.ZipFile(scan_path, "w") as scans,
|
||||||
|
zipfile.ZipFile(label_path, "w") as labels,
|
||||||
|
):
|
||||||
|
for sequence, height in enumerate((1.20, 1.25, 1.30, 1.35, 1.40)):
|
||||||
|
sequence_id = f"{sequence:05d}"
|
||||||
|
point_member = f"{sequence_id}/os1_cloud_node_kitti_bin/000000.bin"
|
||||||
|
label_member = (
|
||||||
|
f"{sequence_id}/os1_cloud_node_semantickitti_label_id/000000.label"
|
||||||
|
)
|
||||||
|
point_bytes, label_bytes = _frame(height)
|
||||||
|
scans.writestr(f"Rellis-3D/{point_member}", point_bytes)
|
||||||
|
labels.writestr(f"Rellis-3D/{label_member}", label_bytes)
|
||||||
|
pairs.append((point_member, label_member))
|
||||||
|
|
||||||
|
result = calibrate_rellis_sensor_height(
|
||||||
|
scan_path,
|
||||||
|
label_path,
|
||||||
|
tuple(pairs),
|
||||||
|
archive_identity="a" * 64,
|
||||||
|
frame_count=5,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["split"] == "train"
|
||||||
|
assert result["validation_labels_used"] is False
|
||||||
|
assert result["frame_count"] == 5
|
||||||
|
assert result["sensor_height_m"] == pytest.approx(1.30)
|
||||||
|
assert len(result["identity_sha256"]) == 64
|
||||||
|
|
||||||
|
|
||||||
|
def test_rellis_archive_index_rejects_path_traversal(tmp_path: Path) -> None:
|
||||||
|
archive = tmp_path / "unsafe.zip"
|
||||||
|
with zipfile.ZipFile(archive, "w") as target:
|
||||||
|
target.writestr("../outside.bin", b"data")
|
||||||
|
with (
|
||||||
|
zipfile.ZipFile(archive) as source,
|
||||||
|
pytest.raises(RellisAdmissionError, match="unsafe"),
|
||||||
|
):
|
||||||
|
_member_index(source)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rellis_patchwork_profile_rejects_unphysical_height() -> None:
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
RellisPatchworkProfile(
|
||||||
|
patchwork_sensor_height_proxy_m=0.1,
|
||||||
|
calibration_identity_sha256="a" * 64,
|
||||||
|
calibration_frame_count=64,
|
||||||
|
calibration_median_absolute_deviation_m=0.01,
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue