feat: qualify complete RELLIS ground dataset
This commit is contained in:
@@ -38,6 +38,22 @@ from k1link.datasets.goose_qualification import (
|
||||
GroundAcceptancePolicy,
|
||||
qualify_goose_ground,
|
||||
)
|
||||
from k1link.datasets.rellis_admission import (
|
||||
RELLIS_ADMISSION_SCHEMA,
|
||||
RellisAdmissionError,
|
||||
admit_rellis_release,
|
||||
)
|
||||
from k1link.datasets.rellis_profile import (
|
||||
RELLIS_PATCHWORK_PROFILE_SCHEMA,
|
||||
RellisPatchworkProfile,
|
||||
)
|
||||
from k1link.datasets.rellis_qualification import (
|
||||
RELLIS_QUALIFICATION_REPORT_SCHEMA,
|
||||
RELLIS_REVIEW_PACK_SCHEMA,
|
||||
RellisAcceptancePolicy,
|
||||
calibrate_rellis_sensor_height,
|
||||
qualify_rellis_ground,
|
||||
)
|
||||
from k1link.datasets.rellis_smoke import (
|
||||
RELLIS_CLASSES,
|
||||
RELLIS_GROUND_POLICY_SCHEMA,
|
||||
@@ -64,9 +80,16 @@ __all__ = [
|
||||
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
|
||||
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
|
||||
"RELLIS_CLASSES",
|
||||
"RELLIS_ADMISSION_SCHEMA",
|
||||
"RELLIS_GROUND_POLICY_SCHEMA",
|
||||
"RELLIS_PATCHWORK_PROFILE_SCHEMA",
|
||||
"RELLIS_PREVIEW_SCHEMA",
|
||||
"RELLIS_QUALIFICATION_REPORT_SCHEMA",
|
||||
"RELLIS_REVIEW_PACK_SCHEMA",
|
||||
"RELLIS_SOURCE_ID",
|
||||
"RellisAcceptancePolicy",
|
||||
"RellisAdmissionError",
|
||||
"RellisPatchworkProfile",
|
||||
"RellisSmokeError",
|
||||
"GoosePatchworkProfile",
|
||||
"GroundAcceptancePolicy",
|
||||
@@ -74,7 +97,9 @@ __all__ = [
|
||||
"DEFAULT_DEGRADATIONS",
|
||||
"benchmark_goose_current_ground",
|
||||
"benchmark_goose_patchwork_ground",
|
||||
"admit_rellis_release",
|
||||
"build_rellis_official_smoke_preview",
|
||||
"calibrate_rellis_sensor_height",
|
||||
"configured_dataset_admission_manifest",
|
||||
"configured_dataset_ground_preview",
|
||||
"configured_dataset_preview",
|
||||
@@ -85,5 +110,6 @@ __all__ = [
|
||||
"read_dataset_native_scan_preview",
|
||||
"read_semantic_kitti_frame",
|
||||
"rellis_native_scan_preview",
|
||||
"qualify_rellis_ground",
|
||||
"qualify_goose_ground",
|
||||
]
|
||||
|
||||
@@ -15,6 +15,8 @@ from k1link.datasets.goose_benchmark import (
|
||||
)
|
||||
from k1link.datasets.goose_qualification import qualify_goose_ground
|
||||
from k1link.datasets.goose_review import build_goose_ground_review_pack
|
||||
from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release
|
||||
from k1link.datasets.rellis_qualification import qualify_rellis_ground
|
||||
from k1link.datasets.rellis_smoke import (
|
||||
RellisSmokeError,
|
||||
build_rellis_official_smoke_preview,
|
||||
@@ -207,5 +209,93 @@ def build_rellis_smoke_preview_command(
|
||||
)
|
||||
|
||||
|
||||
@app.command("admit-rellis")
|
||||
def admit_rellis_command(
|
||||
dataset_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||
],
|
||||
scan_archive: Annotated[
|
||||
Path | None,
|
||||
typer.Option("--scan-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||
] = None,
|
||||
label_archive: Annotated[
|
||||
Path | None,
|
||||
typer.Option("--label-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||
] = None,
|
||||
pose_archive: Annotated[
|
||||
Path | None,
|
||||
typer.Option("--pose-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||
] = None,
|
||||
split_archive: Annotated[
|
||||
Path | None,
|
||||
typer.Option("--split-archive", exists=True, dir_okay=False, resolve_path=True),
|
||||
] = None,
|
||||
preview_points: Annotated[
|
||||
int,
|
||||
typer.Option("--preview-points", min=1, max=20_000),
|
||||
] = 20_000,
|
||||
) -> None:
|
||||
"""Verify the complete official RELLIS release on worker D."""
|
||||
|
||||
try:
|
||||
manifest = admit_rellis_release(
|
||||
dataset_root,
|
||||
scan_archive=scan_archive,
|
||||
label_archive=label_archive,
|
||||
pose_archive=pose_archive,
|
||||
split_archive=split_archive,
|
||||
preview_points=preview_points,
|
||||
)
|
||||
except RellisAdmissionError as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
@app.command("qualify-rellis-ground")
|
||||
def qualify_rellis_ground_command(
|
||||
dataset_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||
],
|
||||
runs_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--runs-root", file_okay=False, resolve_path=True),
|
||||
],
|
||||
mission_core_commit: Annotated[
|
||||
str,
|
||||
typer.Option("--mission-core-commit", min=7, max=64),
|
||||
],
|
||||
workers: Annotated[
|
||||
int,
|
||||
typer.Option("--workers", min=1, max=32),
|
||||
] = 8,
|
||||
calibration_frames: Annotated[
|
||||
int,
|
||||
typer.Option("--calibration-frames", min=5, max=256),
|
||||
] = 64,
|
||||
preview_points: Annotated[
|
||||
int,
|
||||
typer.Option("--preview-points", min=1, max=20_000),
|
||||
] = 12_000,
|
||||
) -> None:
|
||||
"""Calibrate on RELLIS train and qualify Current/Patchwork++ on validation."""
|
||||
|
||||
try:
|
||||
report = qualify_rellis_ground(
|
||||
dataset_root,
|
||||
runs_root,
|
||||
mission_core_commit=mission_core_commit,
|
||||
parallel_workers=workers,
|
||||
calibration_frame_count=calibration_frames,
|
||||
preview_points=preview_points,
|
||||
)
|
||||
except (RellisAdmissionError, ValueError) as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
|
||||
@@ -178,6 +178,7 @@ def dataset_gateway_catalog(
|
||||
dataset_root: Path | None = None,
|
||||
admission_manifest_path: Path | None = None,
|
||||
rellis_preview_path: Path | None = None,
|
||||
rellis_admission_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||
|
||||
@@ -205,6 +206,34 @@ def dataset_gateway_catalog(
|
||||
rellis_preview = candidate
|
||||
except DatasetAdmissionError:
|
||||
pass
|
||||
rellis_admission: dict[str, Any] | None = None
|
||||
if rellis_admission_path is not None and rellis_admission_path.is_file():
|
||||
try:
|
||||
candidate = _bounded_json_object(
|
||||
rellis_admission_path,
|
||||
MAX_STATE_BYTES,
|
||||
"RELLIS admission manifest",
|
||||
)
|
||||
storage = _object(candidate.get("storage"), "RELLIS storage")
|
||||
alignment = _object(candidate.get("alignment"), "RELLIS alignment")
|
||||
release = _object(candidate.get("release"), "RELLIS release")
|
||||
if (
|
||||
candidate.get("schema_version") != "missioncore.rellis-admission/v1"
|
||||
or candidate.get("source_id") != "rellis-3d/v1.1"
|
||||
or candidate.get("status") != "dataset-ready"
|
||||
or storage.get("policy") != "worker-d-only"
|
||||
or storage.get("admitted") is not True
|
||||
or storage.get("path_exposed") is not False
|
||||
or alignment.get("annotated_scan_count") != 13_556
|
||||
or alignment.get("point_label_alignment") != "complete"
|
||||
or alignment.get("pose_index_coverage") != "complete"
|
||||
or not isinstance(release.get("identity_sha256"), str)
|
||||
or len(release["identity_sha256"]) != 64
|
||||
):
|
||||
raise DatasetAdmissionError("RELLIS admission manifest is incompatible")
|
||||
rellis_admission = candidate
|
||||
except DatasetAdmissionError:
|
||||
pass
|
||||
source_status = (
|
||||
str(admission["status"])
|
||||
if admission is not None
|
||||
@@ -222,14 +251,18 @@ def dataset_gateway_catalog(
|
||||
else "configure-dataset-root-on-worker-d"
|
||||
)
|
||||
rellis_status = (
|
||||
"smoke-ready"
|
||||
"dataset-ready"
|
||||
if rellis_admission is not None and rellis_preview is not None
|
||||
else "smoke-ready"
|
||||
if rellis_preview is not None
|
||||
else "ready-for-smoke"
|
||||
if storage_admitted
|
||||
else "blocked-storage-policy"
|
||||
)
|
||||
catalog_next_action = (
|
||||
"admit-rellis-ouster-semantickitti-to-worker-d"
|
||||
"qualify-rellis-current-vs-patchworkpp"
|
||||
if rellis_admission is not None
|
||||
else "admit-rellis-ouster-semantickitti-to-worker-d"
|
||||
if rellis_preview is not None and source_status == "frame-ready"
|
||||
else "build-rellis-official-example-smoke"
|
||||
if source_status == "frame-ready"
|
||||
@@ -342,7 +375,9 @@ def dataset_gateway_catalog(
|
||||
"admission": {
|
||||
"status": rellis_status,
|
||||
"native_scan": (
|
||||
"official-example-compatible"
|
||||
"complete-release-split-aligned"
|
||||
if rellis_admission is not None
|
||||
else "official-example-compatible"
|
||||
if rellis_preview is not None
|
||||
else "requires-official-example-smoke"
|
||||
),
|
||||
|
||||
@@ -0,0 +1,417 @@
|
||||
"""Fail-closed admission of the complete official RELLIS-3D release.
|
||||
|
||||
All four source archives remain on the Simulation Worker D drive. Admission
|
||||
indexes the ZIP central directories, proves split/scan/label/pose alignment and
|
||||
publishes only a path-free manifest plus one bounded validation-frame preview.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.datasets.gateway import DatasetFrameError, decode_semantic_kitti_frame
|
||||
from k1link.datasets.rellis_smoke import (
|
||||
RELLIS_LABEL_CONFIG_SHA256,
|
||||
RELLIS_LICENSE,
|
||||
RELLIS_REPOSITORY_COMMIT,
|
||||
RELLIS_REPOSITORY_URL,
|
||||
RELLIS_SOURCE_ID,
|
||||
RellisSmokeError,
|
||||
rellis_native_scan_preview,
|
||||
)
|
||||
|
||||
RELLIS_ADMISSION_SCHEMA: Final = "missioncore.rellis-admission/v1"
|
||||
RELLIS_RELEASE_ID: Final = "v1.1"
|
||||
RELLIS_ARCHIVE_ROOT: Final = "rellis-3d/v1.1/archives"
|
||||
RELLIS_SCAN_ARCHIVE: Final = "Rellis_3D_os1_cloud_node_kitti_bin.zip"
|
||||
RELLIS_LABEL_ARCHIVE: Final = "Rellis_3D_os1_labels_20210614.zip"
|
||||
RELLIS_POSE_ARCHIVE: Final = "Rellis_3D_lidar_poses_20210614.zip"
|
||||
RELLIS_SPLIT_ARCHIVE: Final = "Rellis_3D_lidar_split.zip"
|
||||
RELLIS_ARCHIVE_URLS: Final[dict[str, str]] = {
|
||||
RELLIS_SCAN_ARCHIVE: (
|
||||
"https://drive.google.com/uc?id=1lDSVRf_kZrD0zHHMsKJ0V1GN9QATR4wH"
|
||||
),
|
||||
RELLIS_LABEL_ARCHIVE: (
|
||||
"https://drive.google.com/uc?id=12bsblHXtob60KrjV7lGXUQTdC5PhV8Er"
|
||||
),
|
||||
RELLIS_POSE_ARCHIVE: (
|
||||
"https://drive.google.com/uc?id=1V3PT_NJhA41N7TBLp5AbW31d0ztQDQOX"
|
||||
),
|
||||
RELLIS_SPLIT_ARCHIVE: (
|
||||
"https://drive.google.com/uc?id=1raQJPySyqDaHpc53KPnJVl3Bln6HlcVS"
|
||||
),
|
||||
}
|
||||
RELLIS_OBSERVED_SHA256: Final[dict[str, str]] = {
|
||||
RELLIS_LABEL_ARCHIVE: (
|
||||
"03297c30b9f2182c74be93f8564d6ce1349237fcc4f89fc96479fe71bb40d905"
|
||||
),
|
||||
RELLIS_POSE_ARCHIVE: (
|
||||
"6deaf38a9cac3a480cfdb70fa9a1d9278f25c36f5efd2487a3c4a6a400250f67"
|
||||
),
|
||||
RELLIS_SPLIT_ARCHIVE: (
|
||||
"de639728e0058d7d477188028d9b811313a601884ef00d267ce13736be7cf54f"
|
||||
),
|
||||
}
|
||||
RELLIS_SPLIT_COUNTS: Final[dict[str, int]] = {
|
||||
"train": 7_800,
|
||||
"validation": 2_413,
|
||||
"test": 3_343,
|
||||
}
|
||||
RELLIS_SEQUENCE_IDS: Final = ("00000", "00001", "00002", "00003", "00004")
|
||||
RELLIS_ANNOTATED_SCAN_COUNT: Final = 13_556
|
||||
MAX_ARCHIVE_ENTRIES: Final = 40_000
|
||||
MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3
|
||||
MAX_METADATA_MEMBER_BYTES: Final = 32 * 1024**2
|
||||
MAX_PREVIEW_POINTS: Final = 20_000
|
||||
_FRAME_PATH = re.compile(
|
||||
r"^(?P<sequence>0000[0-4])/os1_cloud_node_kitti_bin/"
|
||||
r"(?P<frame>[0-9]{6})\.bin$"
|
||||
)
|
||||
_LABEL_PATH = re.compile(
|
||||
r"^(?P<sequence>0000[0-4])/os1_cloud_node_semantickitti_label_id/"
|
||||
r"(?P<frame>[0-9]{6})\.label$"
|
||||
)
|
||||
|
||||
|
||||
class RellisAdmissionError(RuntimeError):
|
||||
"""The official RELLIS release cannot satisfy its pinned admission contract."""
|
||||
|
||||
|
||||
def admit_rellis_release(
|
||||
dataset_root: Path,
|
||||
*,
|
||||
scan_archive: Path | None = None,
|
||||
label_archive: Path | None = None,
|
||||
pose_archive: Path | None = None,
|
||||
split_archive: Path | None = None,
|
||||
preview_points: int = MAX_PREVIEW_POINTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify the full official release without extracting its source archives."""
|
||||
|
||||
root = dataset_root.expanduser().absolute()
|
||||
if not _is_worker_dataset_root(root):
|
||||
raise RellisAdmissionError("RELLIS admission requires the canonical worker D root")
|
||||
if not 1 <= preview_points <= MAX_PREVIEW_POINTS:
|
||||
raise RellisAdmissionError("RELLIS preview point limit is outside the admitted range")
|
||||
archive_root = root / RELLIS_ARCHIVE_ROOT
|
||||
paths = {
|
||||
RELLIS_SCAN_ARCHIVE: _archive_path(archive_root, scan_archive, RELLIS_SCAN_ARCHIVE),
|
||||
RELLIS_LABEL_ARCHIVE: _archive_path(archive_root, label_archive, RELLIS_LABEL_ARCHIVE),
|
||||
RELLIS_POSE_ARCHIVE: _archive_path(archive_root, pose_archive, RELLIS_POSE_ARCHIVE),
|
||||
RELLIS_SPLIT_ARCHIVE: _archive_path(archive_root, split_archive, RELLIS_SPLIT_ARCHIVE),
|
||||
}
|
||||
if any(not path.is_file() for path in paths.values()):
|
||||
raise RellisAdmissionError("one or more official RELLIS archives are unavailable")
|
||||
|
||||
descriptors: dict[str, dict[str, Any]] = {}
|
||||
for filename, path in paths.items():
|
||||
digest = _sha256_file(path)
|
||||
observed = RELLIS_OBSERVED_SHA256.get(filename)
|
||||
if observed is not None and digest != observed:
|
||||
raise RellisAdmissionError(f"{filename} differs from the pinned observed release")
|
||||
descriptors[filename] = {
|
||||
"filename": filename,
|
||||
"source_url": RELLIS_ARCHIVE_URLS[filename],
|
||||
"size_bytes": path.stat().st_size,
|
||||
"sha256": digest,
|
||||
"vendor_checksum_available": False,
|
||||
}
|
||||
|
||||
try:
|
||||
with (
|
||||
zipfile.ZipFile(paths[RELLIS_SCAN_ARCHIVE]) as scans_zip,
|
||||
zipfile.ZipFile(paths[RELLIS_LABEL_ARCHIVE]) as labels_zip,
|
||||
zipfile.ZipFile(paths[RELLIS_POSE_ARCHIVE]) as poses_zip,
|
||||
zipfile.ZipFile(paths[RELLIS_SPLIT_ARCHIVE]) as splits_zip,
|
||||
):
|
||||
scan_members = _member_index(scans_zip)
|
||||
label_members = _member_index(labels_zip)
|
||||
pose_members = _member_index(poses_zip)
|
||||
split_members = _member_index(splits_zip)
|
||||
splits = _read_splits(splits_zip, split_members)
|
||||
split_pairs = {
|
||||
pair for split in splits.values() for pair in split
|
||||
}
|
||||
if len(split_pairs) != RELLIS_ANNOTATED_SCAN_COUNT:
|
||||
raise RellisAdmissionError("RELLIS split union is incomplete or overlapping")
|
||||
|
||||
indexed_scans = {
|
||||
path: member
|
||||
for path, member in scan_members.items()
|
||||
if _FRAME_PATH.fullmatch(path)
|
||||
}
|
||||
indexed_labels = {
|
||||
path: member
|
||||
for path, member in label_members.items()
|
||||
if _LABEL_PATH.fullmatch(path)
|
||||
}
|
||||
expected_scans = {point_path for point_path, _ in split_pairs}
|
||||
expected_labels = {label_path for _, label_path in split_pairs}
|
||||
if set(indexed_scans) != expected_scans:
|
||||
raise RellisAdmissionError("RELLIS scan archive does not match the official splits")
|
||||
if set(indexed_labels) != expected_labels:
|
||||
raise RellisAdmissionError(
|
||||
"RELLIS label archive does not match the official splits"
|
||||
)
|
||||
_validate_member_alignment(indexed_scans, indexed_labels, split_pairs)
|
||||
pose_counts = _validate_poses(poses_zip, pose_members, split_pairs)
|
||||
|
||||
validation_point, validation_label = splits["validation"][0]
|
||||
point_bytes = scans_zip.read(indexed_scans[validation_point])
|
||||
label_bytes = labels_zip.read(indexed_labels[validation_label])
|
||||
frame = decode_semantic_kitti_frame(point_bytes, label_bytes)
|
||||
frame_id = _qualified_frame_id(validation_point)
|
||||
preview = rellis_native_scan_preview(
|
||||
frame,
|
||||
frame_id=frame_id,
|
||||
maximum_points=preview_points,
|
||||
source_evidence={
|
||||
"repository_url": RELLIS_REPOSITORY_URL,
|
||||
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
|
||||
"point_sha256": hashlib.sha256(point_bytes).hexdigest(),
|
||||
"label_sha256": hashlib.sha256(label_bytes).hexdigest(),
|
||||
"license": RELLIS_LICENSE,
|
||||
},
|
||||
)
|
||||
except (OSError, KeyError, zipfile.BadZipFile, DatasetFrameError, RellisSmokeError) as exc:
|
||||
raise RellisAdmissionError("RELLIS archives could not be verified") from exc
|
||||
|
||||
sequence_counts = Counter(
|
||||
_FRAME_PATH.fullmatch(point_path).group("sequence") # type: ignore[union-attr]
|
||||
for point_path, _ in split_pairs
|
||||
)
|
||||
install_identity = hashlib.sha256(
|
||||
json.dumps(descriptors, sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
install_root = root / "rellis-3d/v1.1/installs" / install_identity
|
||||
preview_path = install_root / "previews" / f"{frame_id}.json"
|
||||
_atomic_json(preview_path, preview)
|
||||
manifest = {
|
||||
"schema_version": RELLIS_ADMISSION_SCHEMA,
|
||||
"source_id": RELLIS_SOURCE_ID,
|
||||
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
"status": "dataset-ready",
|
||||
"storage": {
|
||||
"policy": "worker-d-only",
|
||||
"admitted": True,
|
||||
"canonical_root": True,
|
||||
"path_exposed": False,
|
||||
},
|
||||
"release": {
|
||||
"repository_url": RELLIS_REPOSITORY_URL,
|
||||
"repository_commit": RELLIS_REPOSITORY_COMMIT,
|
||||
"license": RELLIS_LICENSE,
|
||||
"archives": descriptors,
|
||||
"identity_sha256": install_identity,
|
||||
},
|
||||
"alignment": {
|
||||
"annotated_scan_count": len(split_pairs),
|
||||
"split_counts": RELLIS_SPLIT_COUNTS,
|
||||
"sequence_counts": dict(sorted(sequence_counts.items())),
|
||||
"pose_counts": pose_counts,
|
||||
"point_label_alignment": "complete",
|
||||
"pose_index_coverage": "complete",
|
||||
},
|
||||
"frame": {
|
||||
"frame_id": frame_id,
|
||||
"split": "validation",
|
||||
"source_point_count": frame.point_count,
|
||||
"preview_point_count": int(preview["point_count"]),
|
||||
"preview_sha256": _sha256_file(preview_path),
|
||||
},
|
||||
"coordinate_frame": {
|
||||
"frame_id": "sensor/lidar/os1",
|
||||
"x": "forward",
|
||||
"y": "left",
|
||||
"z": "up",
|
||||
"transform_applied": False,
|
||||
},
|
||||
"next_action": "calibrate-on-train-then-qualify-validation",
|
||||
}
|
||||
_atomic_json(root / "state/rellis-3d-v1.1.json", manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def read_rellis_splits(dataset_root: Path) -> dict[str, tuple[tuple[str, str], ...]]:
|
||||
"""Read the already admitted immutable split index."""
|
||||
|
||||
root = dataset_root.expanduser().absolute()
|
||||
if not _is_worker_dataset_root(root):
|
||||
raise RellisAdmissionError("RELLIS split access requires the canonical worker D root")
|
||||
split_path = root / RELLIS_ARCHIVE_ROOT / RELLIS_SPLIT_ARCHIVE
|
||||
try:
|
||||
with zipfile.ZipFile(split_path) as source:
|
||||
return _read_splits(source, _member_index(source))
|
||||
except (OSError, zipfile.BadZipFile) as exc:
|
||||
raise RellisAdmissionError("RELLIS split archive cannot be indexed") from exc
|
||||
|
||||
|
||||
def _archive_path(root: Path, value: Path | None, filename: str) -> Path:
|
||||
return value.expanduser().absolute() if value is not None else root / filename
|
||||
|
||||
|
||||
def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]:
|
||||
members = source.infolist()
|
||||
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
|
||||
raise RellisAdmissionError("RELLIS archive entry count is outside the admitted range")
|
||||
total = 0
|
||||
result: dict[str, zipfile.ZipInfo] = {}
|
||||
for member in members:
|
||||
path = PurePosixPath(member.filename.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
raise RellisAdmissionError("RELLIS archive contains an unsafe member path")
|
||||
if (member.external_attr >> 16) & 0o170000 == 0o120000:
|
||||
raise RellisAdmissionError("RELLIS archive contains a symbolic link")
|
||||
total += member.file_size
|
||||
if total > MAX_UNCOMPRESSED_BYTES:
|
||||
raise RellisAdmissionError("RELLIS archive exceeds the uncompressed safety limit")
|
||||
if member.is_dir():
|
||||
continue
|
||||
normalized = "/".join(part for part in path.parts if part not in {".", "Rellis-3D"})
|
||||
if not normalized or normalized in result:
|
||||
raise RellisAdmissionError("RELLIS archive member identity is ambiguous")
|
||||
result[normalized] = member
|
||||
return result
|
||||
|
||||
|
||||
def _read_splits(
|
||||
source: zipfile.ZipFile,
|
||||
members: dict[str, zipfile.ZipInfo],
|
||||
) -> dict[str, tuple[tuple[str, str], ...]]:
|
||||
result: dict[str, tuple[tuple[str, str], ...]] = {}
|
||||
for split, expected in RELLIS_SPLIT_COUNTS.items():
|
||||
filename = "pt_val.lst" if split == "validation" else f"pt_{split}.lst"
|
||||
matches = [member for path, member in members.items() if path.endswith(filename)]
|
||||
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
|
||||
raise RellisAdmissionError(f"RELLIS {split} split index is unavailable")
|
||||
try:
|
||||
lines = source.read(matches[0]).decode("utf-8").splitlines()
|
||||
except UnicodeDecodeError as exc:
|
||||
raise RellisAdmissionError("RELLIS split index is not UTF-8") from exc
|
||||
pairs: list[tuple[str, str]] = []
|
||||
for raw in lines:
|
||||
parts = raw.strip().replace("\\", "/").split()
|
||||
if len(parts) != 2:
|
||||
raise RellisAdmissionError("RELLIS split row is malformed")
|
||||
point_path, label_path = parts
|
||||
point_match = _FRAME_PATH.fullmatch(point_path)
|
||||
label_match = _LABEL_PATH.fullmatch(label_path)
|
||||
if (
|
||||
point_match is None
|
||||
or label_match is None
|
||||
or point_match.group("sequence") != label_match.group("sequence")
|
||||
or point_match.group("frame") != label_match.group("frame")
|
||||
):
|
||||
raise RellisAdmissionError("RELLIS split point and label identities differ")
|
||||
pairs.append((point_path, label_path))
|
||||
if len(pairs) != expected or len(set(pairs)) != expected:
|
||||
raise RellisAdmissionError(f"RELLIS {split} split count differs")
|
||||
result[split] = tuple(pairs)
|
||||
if sum(map(len, result.values())) != len(
|
||||
{pair for values in result.values() for pair in values}
|
||||
):
|
||||
raise RellisAdmissionError("RELLIS split indexes overlap")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_member_alignment(
|
||||
scans: dict[str, zipfile.ZipInfo],
|
||||
labels: dict[str, zipfile.ZipInfo],
|
||||
pairs: set[tuple[str, str]],
|
||||
) -> None:
|
||||
for point_path, label_path in pairs:
|
||||
point = scans[point_path]
|
||||
label = labels[label_path]
|
||||
if (
|
||||
point.file_size <= 0
|
||||
or point.file_size % 16
|
||||
or label.file_size % 4
|
||||
or point.file_size // 16 != label.file_size // 4
|
||||
):
|
||||
raise RellisAdmissionError("RELLIS point and label member sizes are not aligned")
|
||||
|
||||
|
||||
def _validate_poses(
|
||||
source: zipfile.ZipFile,
|
||||
members: dict[str, zipfile.ZipInfo],
|
||||
pairs: set[tuple[str, str]],
|
||||
) -> dict[str, int]:
|
||||
result: dict[str, int] = {}
|
||||
required_by_sequence: dict[str, set[int]] = {
|
||||
sequence: set() for sequence in RELLIS_SEQUENCE_IDS
|
||||
}
|
||||
for point_path, _ in pairs:
|
||||
match = _FRAME_PATH.fullmatch(point_path)
|
||||
assert match is not None
|
||||
required_by_sequence[match.group("sequence")].add(int(match.group("frame")))
|
||||
for sequence in RELLIS_SEQUENCE_IDS:
|
||||
path = f"{sequence}/poses.txt"
|
||||
matches = [member for name, member in members.items() if name.endswith(path)]
|
||||
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
|
||||
raise RellisAdmissionError(f"RELLIS poses are unavailable for sequence {sequence}")
|
||||
try:
|
||||
rows = source.read(matches[0]).decode("utf-8").splitlines()
|
||||
except UnicodeDecodeError as exc:
|
||||
raise RellisAdmissionError("RELLIS pose file is not UTF-8") from exc
|
||||
for row in rows:
|
||||
values = row.split()
|
||||
if len(values) != 12:
|
||||
raise RellisAdmissionError("RELLIS pose row is not a 3x4 transform")
|
||||
try:
|
||||
tuple(float(value) for value in values)
|
||||
except ValueError as exc:
|
||||
raise RellisAdmissionError("RELLIS pose row is not numeric") from exc
|
||||
if not rows or max(required_by_sequence[sequence], default=-1) >= len(rows):
|
||||
raise RellisAdmissionError("RELLIS pose index does not cover every split frame")
|
||||
result[sequence] = len(rows)
|
||||
return result
|
||||
|
||||
|
||||
def _qualified_frame_id(point_path: str) -> str:
|
||||
match = _FRAME_PATH.fullmatch(point_path)
|
||||
if match is None:
|
||||
raise RellisAdmissionError("RELLIS frame path is incompatible")
|
||||
return f"{match.group('sequence')}-{match.group('frame')}"
|
||||
|
||||
|
||||
def _is_worker_dataset_root(root: Path) -> bool:
|
||||
return str(root).replace("\\", "/").rstrip("/").lower() == (
|
||||
"/mnt/d/ndc_missioncore/datasets"
|
||||
)
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
raise RellisAdmissionError("RELLIS artifact cannot be hashed") from exc
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
dir=path.parent,
|
||||
encoding="utf-8",
|
||||
delete=False,
|
||||
) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
json.dump(document, temporary, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
temporary.write("\n")
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Train-calibrated, algorithm-scoped RELLIS OS1 Patchwork++ profile."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Final
|
||||
|
||||
RELLIS_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.rellis-patchwork-profile/v1"
|
||||
RELLIS_URDF_URL: Final = (
|
||||
"https://raw.githubusercontent.com/unmannedlab/RELLIS-3D/"
|
||||
"c17a118fcaed1559f03cc32cc3a91dedc557f8b8/"
|
||||
"catkin_ws/src/platform_description/urdf/warthog.urdf.xacro"
|
||||
)
|
||||
RELLIS_URDF_SHA256: Final = (
|
||||
"bcb66a05b8818f06479bb4a17ce93e9d66cb8fdaab726c437596807288116ac3"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RellisPatchworkProfile:
|
||||
"""Frozen provider profile derived without looking at validation labels."""
|
||||
|
||||
patchwork_sensor_height_proxy_m: float
|
||||
calibration_identity_sha256: str
|
||||
calibration_frame_count: int
|
||||
calibration_median_absolute_deviation_m: float
|
||||
patchwork_minimum_range_m: float = 2.7
|
||||
patchwork_maximum_range_m: float = 80.0
|
||||
profile_id: str = "rellis-warthog-os1-patchwork/v1"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
values = (
|
||||
self.patchwork_sensor_height_proxy_m,
|
||||
self.calibration_median_absolute_deviation_m,
|
||||
self.patchwork_minimum_range_m,
|
||||
self.patchwork_maximum_range_m,
|
||||
)
|
||||
if (
|
||||
not self.profile_id
|
||||
or not all(math.isfinite(value) for value in values)
|
||||
or not 0.5 <= self.patchwork_sensor_height_proxy_m <= 3.0
|
||||
or not 5 <= self.calibration_frame_count <= 256
|
||||
or self.calibration_median_absolute_deviation_m < 0
|
||||
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
|
||||
or len(self.calibration_identity_sha256) != 64
|
||||
or any(
|
||||
character not in "0123456789abcdef"
|
||||
for character in self.calibration_identity_sha256
|
||||
)
|
||||
):
|
||||
raise ValueError("RELLIS Patchwork++ profile is invalid")
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": RELLIS_PATCHWORK_PROFILE_SCHEMA,
|
||||
"profile_id": self.profile_id,
|
||||
"source_id": "rellis-3d/v1.1",
|
||||
"representation": "native-scan",
|
||||
"sensor_frame": {
|
||||
"frame_id": "sensor/lidar/os1",
|
||||
"handedness": "right",
|
||||
"x": "forward",
|
||||
"y": "left",
|
||||
"z": "up",
|
||||
"one_revolution": True,
|
||||
},
|
||||
"height": {
|
||||
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
|
||||
"evidence": "train-split-ground-label-calibration",
|
||||
"calibration_frame_count": self.calibration_frame_count,
|
||||
"median_absolute_deviation_m": (
|
||||
self.calibration_median_absolute_deviation_m
|
||||
),
|
||||
"calibration_identity_sha256": self.calibration_identity_sha256,
|
||||
"validation_labels_used": False,
|
||||
},
|
||||
"patchworkpp": {
|
||||
"provider_id": "patchworkpp/v1.4.1",
|
||||
"minimum_range_m": self.patchwork_minimum_range_m,
|
||||
"maximum_range_m": self.patchwork_maximum_range_m,
|
||||
"enable_rnr": True,
|
||||
"enable_rvpf": True,
|
||||
"enable_tgr": True,
|
||||
},
|
||||
"evidence": {
|
||||
"vehicle_urdf": {
|
||||
"url": RELLIS_URDF_URL,
|
||||
"sha256": RELLIS_URDF_SHA256,
|
||||
"use": "topology-cross-check-only",
|
||||
"exact_physical_height_claimed": False,
|
||||
}
|
||||
},
|
||||
"scope": {
|
||||
"patchworkpp_eligible": True,
|
||||
"normalized_scan_produced": False,
|
||||
"complete_vehicle_transform_known": False,
|
||||
"deskew_claimed": False,
|
||||
},
|
||||
"authority": {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -429,6 +429,9 @@ app.include_router(
|
||||
dataset_rellis_preview_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
|
||||
),
|
||||
dataset_rellis_admission_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-admission.json"
|
||||
),
|
||||
dataset_ground_preview_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
|
||||
),
|
||||
|
||||
@@ -64,6 +64,7 @@ def build_lidar_router(
|
||||
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
|
||||
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
|
||||
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
|
||||
dataset_rellis_admission_provider: DatasetArtifactProvider = lambda: None,
|
||||
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
|
||||
) -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||
@@ -73,6 +74,7 @@ def build_lidar_router(
|
||||
return dataset_gateway_catalog(
|
||||
admission_manifest_path=dataset_admission_provider(),
|
||||
rellis_preview_path=dataset_rellis_preview_provider(),
|
||||
rellis_admission_path=dataset_rellis_admission_provider(),
|
||||
)
|
||||
|
||||
@router.get("/dataset-gateway/preview")
|
||||
|
||||
@@ -41,17 +41,21 @@ DETAIL_SCHEMA: Final = "missioncore.polygon-run-detail/v1"
|
||||
GROUND_QUALIFICATION_SCHEMA: Final = "missioncore.polygon-ground-qualification/v1"
|
||||
GROUND_FAILURE_PREVIEW_SCHEMA: Final = "missioncore.polygon-ground-failure-preview/v1"
|
||||
GROUND_REPORT_ARTIFACT_KIND: Final = "goose-ground-qualification-report"
|
||||
RELLIS_REPORT_ARTIFACT_KIND: Final = "rellis-ground-qualification-report"
|
||||
GROUND_FAILURE_ARTIFACT_KIND: Final = "goose-ground-qualification-failure-preview"
|
||||
GROUND_REPORT_SCHEMA: Final = "missioncore.goose-ground-qualification-report/v1"
|
||||
RELLIS_REPORT_SCHEMA: Final = "missioncore.rellis-ground-qualification-report/v1"
|
||||
GROUND_FAILURE_SCHEMA: Final = "missioncore.goose-ground-qualification-failure-preview/v1"
|
||||
GROUND_REVIEW_SCHEMA: Final = "missioncore.polygon-ground-review/v2"
|
||||
GROUND_REVIEW_FRAME_SCHEMA: Final = "missioncore.polygon-ground-review-frame/v1"
|
||||
GOOSE_REVIEW_PACK_SCHEMA: Final = "missioncore.goose-ground-review-pack/v1"
|
||||
GOOSE_REVIEW_FRAME_SCHEMA: Final = "missioncore.goose-ground-review-frame/v1"
|
||||
RELLIS_REVIEW_PACK_SCHEMA: Final = "missioncore.rellis-ground-review-pack/v1"
|
||||
RELLIS_REVIEW_FRAME_SCHEMA: Final = "missioncore.rellis-ground-review-frame/v1"
|
||||
MAX_QUALIFICATION_ARTIFACT_BYTES: Final = 32 * 1024**2
|
||||
MAX_REVIEW_MANIFEST_BYTES: Final = 8 * 1024**2
|
||||
MAX_REVIEW_FRAME_BYTES: Final = 2 * 1024**2
|
||||
MAX_REVIEW_FRAMES: Final = 2_000
|
||||
MAX_REVIEW_FRAMES: Final = 3_000
|
||||
MAX_REVIEW_POINTS: Final = 20_000
|
||||
COMMIT_PATTERN: Final = re.compile(r"^[a-f0-9]{40}$")
|
||||
GOOSE_FRAME_ID_PATTERN: Final = re.compile(
|
||||
@@ -201,8 +205,8 @@ def build_polygon_router(
|
||||
report = _read_registered_json(
|
||||
store,
|
||||
run,
|
||||
artifact_kind=GROUND_REPORT_ARTIFACT_KIND,
|
||||
expected_schema=GROUND_REPORT_SCHEMA,
|
||||
artifact_kind=(GROUND_REPORT_ARTIFACT_KIND, RELLIS_REPORT_ARTIFACT_KIND),
|
||||
expected_schema=(GROUND_REPORT_SCHEMA, RELLIS_REPORT_SCHEMA),
|
||||
)
|
||||
required = (
|
||||
"identity_sha256",
|
||||
@@ -305,7 +309,7 @@ def build_polygon_router(
|
||||
"ground_iou_delta",
|
||||
)
|
||||
},
|
||||
**_goose_frame_identity(frame["frame_id"]),
|
||||
**_dataset_frame_identity(frame),
|
||||
}
|
||||
for frame in manifest["frames"]
|
||||
],
|
||||
@@ -348,7 +352,11 @@ def build_polygon_router(
|
||||
) from exc
|
||||
point_count = int(frame["point_count"])
|
||||
if (
|
||||
schema != GOOSE_REVIEW_FRAME_SCHEMA
|
||||
schema
|
||||
not in {
|
||||
GOOSE_REVIEW_FRAME_SCHEMA,
|
||||
RELLIS_REVIEW_FRAME_SCHEMA,
|
||||
}
|
||||
or stored_frame_id != frame_id
|
||||
or source_point_count != frame["source_point_count"]
|
||||
or xyz_cm.dtype != np.dtype("<i2")
|
||||
@@ -562,11 +570,17 @@ def _read_registered_json(
|
||||
store: QualificationRunStore,
|
||||
run: QualificationRun,
|
||||
*,
|
||||
artifact_kind: str,
|
||||
expected_schema: str,
|
||||
artifact_kind: str | tuple[str, ...],
|
||||
expected_schema: str | tuple[str, ...],
|
||||
frame_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
candidates = [artifact for artifact in run.artifacts if artifact.kind == artifact_kind]
|
||||
artifact_kinds = (artifact_kind,) if isinstance(artifact_kind, str) else artifact_kind
|
||||
expected_schemas = (
|
||||
(expected_schema,) if isinstance(expected_schema, str) else expected_schema
|
||||
)
|
||||
candidates = [
|
||||
artifact for artifact in run.artifacts if artifact.kind in artifact_kinds
|
||||
]
|
||||
if frame_id is not None:
|
||||
candidates = [
|
||||
artifact for artifact in candidates if Path(artifact.relative_path).stem == frame_id
|
||||
@@ -611,7 +625,7 @@ def _read_registered_json(
|
||||
status_code=500,
|
||||
detail="Квалификационный артефакт не является допустимым JSON.",
|
||||
) from exc
|
||||
if not isinstance(value, dict) or value.get("schema_version") != expected_schema:
|
||||
if not isinstance(value, dict) or value.get("schema_version") not in expected_schemas:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Квалификационный артефакт имеет неизвестную схему.",
|
||||
@@ -676,7 +690,8 @@ def _read_ground_review_manifest(
|
||||
safety = manifest.get("safety") if isinstance(manifest, dict) else None
|
||||
if (
|
||||
not isinstance(manifest, dict)
|
||||
or manifest.get("schema_version") != GOOSE_REVIEW_PACK_SCHEMA
|
||||
or manifest.get("schema_version")
|
||||
not in {GOOSE_REVIEW_PACK_SCHEMA, RELLIS_REVIEW_PACK_SCHEMA}
|
||||
or manifest.get("source_run_id") != run_id
|
||||
or not isinstance(manifest.get("identity_sha256"), str)
|
||||
or not re.fullmatch(r"[a-f0-9]{64}", manifest["identity_sha256"])
|
||||
@@ -725,7 +740,27 @@ def _read_ground_review_manifest(
|
||||
return pack_root, manifest
|
||||
|
||||
|
||||
def _goose_frame_identity(frame_id: str) -> dict[str, str | int]:
|
||||
def _dataset_frame_identity(frame: dict[str, Any]) -> dict[str, str | int | None]:
|
||||
frame_id = frame["frame_id"]
|
||||
explicit_sequence = frame.get("dataset_sequence_id")
|
||||
explicit_number = frame.get("dataset_frame_number")
|
||||
if explicit_sequence is not None or explicit_number is not None:
|
||||
if (
|
||||
not isinstance(explicit_sequence, str)
|
||||
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,95}", explicit_sequence)
|
||||
is None
|
||||
or not isinstance(explicit_number, int)
|
||||
or explicit_number < 0
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Идентичность dataset-кадра нарушает контракт.",
|
||||
)
|
||||
return {
|
||||
"dataset_sequence_id": explicit_sequence,
|
||||
"dataset_frame_number": explicit_number,
|
||||
"sensor_timestamp_ns": None,
|
||||
}
|
||||
match = GOOSE_FRAME_ID_PATTERN.fullmatch(frame_id)
|
||||
if match is None:
|
||||
raise HTTPException(
|
||||
|
||||
Reference in New Issue
Block a user