feat: qualify complete RELLIS ground dataset

This commit is contained in:
DCCONSTRUCTIONS
2026-07-25 21:12:37 +03:00
parent 054feec0d2
commit f2f044080f
17 changed files with 2349 additions and 35 deletions
+26
View File
@@ -38,6 +38,22 @@ from k1link.datasets.goose_qualification import (
GroundAcceptancePolicy,
qualify_goose_ground,
)
from k1link.datasets.rellis_admission import (
RELLIS_ADMISSION_SCHEMA,
RellisAdmissionError,
admit_rellis_release,
)
from k1link.datasets.rellis_profile import (
RELLIS_PATCHWORK_PROFILE_SCHEMA,
RellisPatchworkProfile,
)
from k1link.datasets.rellis_qualification import (
RELLIS_QUALIFICATION_REPORT_SCHEMA,
RELLIS_REVIEW_PACK_SCHEMA,
RellisAcceptancePolicy,
calibrate_rellis_sensor_height,
qualify_rellis_ground,
)
from k1link.datasets.rellis_smoke import (
RELLIS_CLASSES,
RELLIS_GROUND_POLICY_SCHEMA,
@@ -64,9 +80,16 @@ __all__ = [
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
"RELLIS_CLASSES",
"RELLIS_ADMISSION_SCHEMA",
"RELLIS_GROUND_POLICY_SCHEMA",
"RELLIS_PATCHWORK_PROFILE_SCHEMA",
"RELLIS_PREVIEW_SCHEMA",
"RELLIS_QUALIFICATION_REPORT_SCHEMA",
"RELLIS_REVIEW_PACK_SCHEMA",
"RELLIS_SOURCE_ID",
"RellisAcceptancePolicy",
"RellisAdmissionError",
"RellisPatchworkProfile",
"RellisSmokeError",
"GoosePatchworkProfile",
"GroundAcceptancePolicy",
@@ -74,7 +97,9 @@ __all__ = [
"DEFAULT_DEGRADATIONS",
"benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground",
"admit_rellis_release",
"build_rellis_official_smoke_preview",
"calibrate_rellis_sensor_height",
"configured_dataset_admission_manifest",
"configured_dataset_ground_preview",
"configured_dataset_preview",
@@ -85,5 +110,6 @@ __all__ = [
"read_dataset_native_scan_preview",
"read_semantic_kitti_frame",
"rellis_native_scan_preview",
"qualify_rellis_ground",
"qualify_goose_ground",
]
+90
View File
@@ -15,6 +15,8 @@ from k1link.datasets.goose_benchmark import (
)
from k1link.datasets.goose_qualification import qualify_goose_ground
from k1link.datasets.goose_review import build_goose_ground_review_pack
from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release
from k1link.datasets.rellis_qualification import qualify_rellis_ground
from k1link.datasets.rellis_smoke import (
RellisSmokeError,
build_rellis_official_smoke_preview,
@@ -207,5 +209,93 @@ def build_rellis_smoke_preview_command(
)
@app.command("admit-rellis")
def admit_rellis_command(
dataset_root: Annotated[
Path,
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
],
scan_archive: Annotated[
Path | None,
typer.Option("--scan-archive", exists=True, dir_okay=False, resolve_path=True),
] = None,
label_archive: Annotated[
Path | None,
typer.Option("--label-archive", exists=True, dir_okay=False, resolve_path=True),
] = None,
pose_archive: Annotated[
Path | None,
typer.Option("--pose-archive", exists=True, dir_okay=False, resolve_path=True),
] = None,
split_archive: Annotated[
Path | None,
typer.Option("--split-archive", exists=True, dir_okay=False, resolve_path=True),
] = None,
preview_points: Annotated[
int,
typer.Option("--preview-points", min=1, max=20_000),
] = 20_000,
) -> None:
"""Verify the complete official RELLIS release on worker D."""
try:
manifest = admit_rellis_release(
dataset_root,
scan_archive=scan_archive,
label_archive=label_archive,
pose_archive=pose_archive,
split_archive=split_archive,
preview_points=preview_points,
)
except RellisAdmissionError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
@app.command("qualify-rellis-ground")
def qualify_rellis_ground_command(
dataset_root: Annotated[
Path,
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
],
runs_root: Annotated[
Path,
typer.Option("--runs-root", file_okay=False, resolve_path=True),
],
mission_core_commit: Annotated[
str,
typer.Option("--mission-core-commit", min=7, max=64),
],
workers: Annotated[
int,
typer.Option("--workers", min=1, max=32),
] = 8,
calibration_frames: Annotated[
int,
typer.Option("--calibration-frames", min=5, max=256),
] = 64,
preview_points: Annotated[
int,
typer.Option("--preview-points", min=1, max=20_000),
] = 12_000,
) -> None:
"""Calibrate on RELLIS train and qualify Current/Patchwork++ on validation."""
try:
report = qualify_rellis_ground(
dataset_root,
runs_root,
mission_core_commit=mission_core_commit,
parallel_workers=workers,
calibration_frame_count=calibration_frames,
preview_points=preview_points,
)
except (RellisAdmissionError, ValueError) as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
if __name__ == "__main__":
app()
+38 -3
View File
@@ -178,6 +178,7 @@ def dataset_gateway_catalog(
dataset_root: Path | None = None,
admission_manifest_path: Path | None = None,
rellis_preview_path: Path | None = None,
rellis_admission_path: Path | None = None,
) -> dict[str, object]:
"""Return the path-free, read-only ingress plan and current admission state."""
@@ -205,6 +206,34 @@ def dataset_gateway_catalog(
rellis_preview = candidate
except DatasetAdmissionError:
pass
rellis_admission: dict[str, Any] | None = None
if rellis_admission_path is not None and rellis_admission_path.is_file():
try:
candidate = _bounded_json_object(
rellis_admission_path,
MAX_STATE_BYTES,
"RELLIS admission manifest",
)
storage = _object(candidate.get("storage"), "RELLIS storage")
alignment = _object(candidate.get("alignment"), "RELLIS alignment")
release = _object(candidate.get("release"), "RELLIS release")
if (
candidate.get("schema_version") != "missioncore.rellis-admission/v1"
or candidate.get("source_id") != "rellis-3d/v1.1"
or candidate.get("status") != "dataset-ready"
or storage.get("policy") != "worker-d-only"
or storage.get("admitted") is not True
or storage.get("path_exposed") is not False
or alignment.get("annotated_scan_count") != 13_556
or alignment.get("point_label_alignment") != "complete"
or alignment.get("pose_index_coverage") != "complete"
or not isinstance(release.get("identity_sha256"), str)
or len(release["identity_sha256"]) != 64
):
raise DatasetAdmissionError("RELLIS admission manifest is incompatible")
rellis_admission = candidate
except DatasetAdmissionError:
pass
source_status = (
str(admission["status"])
if admission is not None
@@ -222,14 +251,18 @@ def dataset_gateway_catalog(
else "configure-dataset-root-on-worker-d"
)
rellis_status = (
"smoke-ready"
"dataset-ready"
if rellis_admission is not None and rellis_preview is not None
else "smoke-ready"
if rellis_preview is not None
else "ready-for-smoke"
if storage_admitted
else "blocked-storage-policy"
)
catalog_next_action = (
"admit-rellis-ouster-semantickitti-to-worker-d"
"qualify-rellis-current-vs-patchworkpp"
if rellis_admission is not None
else "admit-rellis-ouster-semantickitti-to-worker-d"
if rellis_preview is not None and source_status == "frame-ready"
else "build-rellis-official-example-smoke"
if source_status == "frame-ready"
@@ -342,7 +375,9 @@ def dataset_gateway_catalog(
"admission": {
"status": rellis_status,
"native_scan": (
"official-example-compatible"
"complete-release-split-aligned"
if rellis_admission is not None
else "official-example-compatible"
if rellis_preview is not None
else "requires-official-example-smoke"
),
+417
View File
@@ -0,0 +1,417 @@
"""Fail-closed admission of the complete official RELLIS-3D release.
All four source archives remain on the Simulation Worker D drive. Admission
indexes the ZIP central directories, proves split/scan/label/pose alignment and
publishes only a path-free manifest plus one bounded validation-frame preview.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
import zipfile
from collections import Counter
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Final
from k1link.datasets.gateway import DatasetFrameError, decode_semantic_kitti_frame
from k1link.datasets.rellis_smoke import (
RELLIS_LABEL_CONFIG_SHA256,
RELLIS_LICENSE,
RELLIS_REPOSITORY_COMMIT,
RELLIS_REPOSITORY_URL,
RELLIS_SOURCE_ID,
RellisSmokeError,
rellis_native_scan_preview,
)
RELLIS_ADMISSION_SCHEMA: Final = "missioncore.rellis-admission/v1"
RELLIS_RELEASE_ID: Final = "v1.1"
RELLIS_ARCHIVE_ROOT: Final = "rellis-3d/v1.1/archives"
RELLIS_SCAN_ARCHIVE: Final = "Rellis_3D_os1_cloud_node_kitti_bin.zip"
RELLIS_LABEL_ARCHIVE: Final = "Rellis_3D_os1_labels_20210614.zip"
RELLIS_POSE_ARCHIVE: Final = "Rellis_3D_lidar_poses_20210614.zip"
RELLIS_SPLIT_ARCHIVE: Final = "Rellis_3D_lidar_split.zip"
RELLIS_ARCHIVE_URLS: Final[dict[str, str]] = {
RELLIS_SCAN_ARCHIVE: (
"https://drive.google.com/uc?id=1lDSVRf_kZrD0zHHMsKJ0V1GN9QATR4wH"
),
RELLIS_LABEL_ARCHIVE: (
"https://drive.google.com/uc?id=12bsblHXtob60KrjV7lGXUQTdC5PhV8Er"
),
RELLIS_POSE_ARCHIVE: (
"https://drive.google.com/uc?id=1V3PT_NJhA41N7TBLp5AbW31d0ztQDQOX"
),
RELLIS_SPLIT_ARCHIVE: (
"https://drive.google.com/uc?id=1raQJPySyqDaHpc53KPnJVl3Bln6HlcVS"
),
}
RELLIS_OBSERVED_SHA256: Final[dict[str, str]] = {
RELLIS_LABEL_ARCHIVE: (
"03297c30b9f2182c74be93f8564d6ce1349237fcc4f89fc96479fe71bb40d905"
),
RELLIS_POSE_ARCHIVE: (
"6deaf38a9cac3a480cfdb70fa9a1d9278f25c36f5efd2487a3c4a6a400250f67"
),
RELLIS_SPLIT_ARCHIVE: (
"de639728e0058d7d477188028d9b811313a601884ef00d267ce13736be7cf54f"
),
}
RELLIS_SPLIT_COUNTS: Final[dict[str, int]] = {
"train": 7_800,
"validation": 2_413,
"test": 3_343,
}
RELLIS_SEQUENCE_IDS: Final = ("00000", "00001", "00002", "00003", "00004")
RELLIS_ANNOTATED_SCAN_COUNT: Final = 13_556
MAX_ARCHIVE_ENTRIES: Final = 40_000
MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3
MAX_METADATA_MEMBER_BYTES: Final = 32 * 1024**2
MAX_PREVIEW_POINTS: Final = 20_000
_FRAME_PATH = re.compile(
r"^(?P<sequence>0000[0-4])/os1_cloud_node_kitti_bin/"
r"(?P<frame>[0-9]{6})\.bin$"
)
_LABEL_PATH = re.compile(
r"^(?P<sequence>0000[0-4])/os1_cloud_node_semantickitti_label_id/"
r"(?P<frame>[0-9]{6})\.label$"
)
class RellisAdmissionError(RuntimeError):
"""The official RELLIS release cannot satisfy its pinned admission contract."""
def admit_rellis_release(
dataset_root: Path,
*,
scan_archive: Path | None = None,
label_archive: Path | None = None,
pose_archive: Path | None = None,
split_archive: Path | None = None,
preview_points: int = MAX_PREVIEW_POINTS,
) -> dict[str, Any]:
"""Verify the full official release without extracting its source archives."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise RellisAdmissionError("RELLIS admission requires the canonical worker D root")
if not 1 <= preview_points <= MAX_PREVIEW_POINTS:
raise RellisAdmissionError("RELLIS preview point limit is outside the admitted range")
archive_root = root / RELLIS_ARCHIVE_ROOT
paths = {
RELLIS_SCAN_ARCHIVE: _archive_path(archive_root, scan_archive, RELLIS_SCAN_ARCHIVE),
RELLIS_LABEL_ARCHIVE: _archive_path(archive_root, label_archive, RELLIS_LABEL_ARCHIVE),
RELLIS_POSE_ARCHIVE: _archive_path(archive_root, pose_archive, RELLIS_POSE_ARCHIVE),
RELLIS_SPLIT_ARCHIVE: _archive_path(archive_root, split_archive, RELLIS_SPLIT_ARCHIVE),
}
if any(not path.is_file() for path in paths.values()):
raise RellisAdmissionError("one or more official RELLIS archives are unavailable")
descriptors: dict[str, dict[str, Any]] = {}
for filename, path in paths.items():
digest = _sha256_file(path)
observed = RELLIS_OBSERVED_SHA256.get(filename)
if observed is not None and digest != observed:
raise RellisAdmissionError(f"{filename} differs from the pinned observed release")
descriptors[filename] = {
"filename": filename,
"source_url": RELLIS_ARCHIVE_URLS[filename],
"size_bytes": path.stat().st_size,
"sha256": digest,
"vendor_checksum_available": False,
}
try:
with (
zipfile.ZipFile(paths[RELLIS_SCAN_ARCHIVE]) as scans_zip,
zipfile.ZipFile(paths[RELLIS_LABEL_ARCHIVE]) as labels_zip,
zipfile.ZipFile(paths[RELLIS_POSE_ARCHIVE]) as poses_zip,
zipfile.ZipFile(paths[RELLIS_SPLIT_ARCHIVE]) as splits_zip,
):
scan_members = _member_index(scans_zip)
label_members = _member_index(labels_zip)
pose_members = _member_index(poses_zip)
split_members = _member_index(splits_zip)
splits = _read_splits(splits_zip, split_members)
split_pairs = {
pair for split in splits.values() for pair in split
}
if len(split_pairs) != RELLIS_ANNOTATED_SCAN_COUNT:
raise RellisAdmissionError("RELLIS split union is incomplete or overlapping")
indexed_scans = {
path: member
for path, member in scan_members.items()
if _FRAME_PATH.fullmatch(path)
}
indexed_labels = {
path: member
for path, member in label_members.items()
if _LABEL_PATH.fullmatch(path)
}
expected_scans = {point_path for point_path, _ in split_pairs}
expected_labels = {label_path for _, label_path in split_pairs}
if set(indexed_scans) != expected_scans:
raise RellisAdmissionError("RELLIS scan archive does not match the official splits")
if set(indexed_labels) != expected_labels:
raise RellisAdmissionError(
"RELLIS label archive does not match the official splits"
)
_validate_member_alignment(indexed_scans, indexed_labels, split_pairs)
pose_counts = _validate_poses(poses_zip, pose_members, split_pairs)
validation_point, validation_label = splits["validation"][0]
point_bytes = scans_zip.read(indexed_scans[validation_point])
label_bytes = labels_zip.read(indexed_labels[validation_label])
frame = decode_semantic_kitti_frame(point_bytes, label_bytes)
frame_id = _qualified_frame_id(validation_point)
preview = rellis_native_scan_preview(
frame,
frame_id=frame_id,
maximum_points=preview_points,
source_evidence={
"repository_url": RELLIS_REPOSITORY_URL,
"repository_commit": RELLIS_REPOSITORY_COMMIT,
"label_config_sha256": RELLIS_LABEL_CONFIG_SHA256,
"point_sha256": hashlib.sha256(point_bytes).hexdigest(),
"label_sha256": hashlib.sha256(label_bytes).hexdigest(),
"license": RELLIS_LICENSE,
},
)
except (OSError, KeyError, zipfile.BadZipFile, DatasetFrameError, RellisSmokeError) as exc:
raise RellisAdmissionError("RELLIS archives could not be verified") from exc
sequence_counts = Counter(
_FRAME_PATH.fullmatch(point_path).group("sequence") # type: ignore[union-attr]
for point_path, _ in split_pairs
)
install_identity = hashlib.sha256(
json.dumps(descriptors, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
install_root = root / "rellis-3d/v1.1/installs" / install_identity
preview_path = install_root / "previews" / f"{frame_id}.json"
_atomic_json(preview_path, preview)
manifest = {
"schema_version": RELLIS_ADMISSION_SCHEMA,
"source_id": RELLIS_SOURCE_ID,
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"status": "dataset-ready",
"storage": {
"policy": "worker-d-only",
"admitted": True,
"canonical_root": True,
"path_exposed": False,
},
"release": {
"repository_url": RELLIS_REPOSITORY_URL,
"repository_commit": RELLIS_REPOSITORY_COMMIT,
"license": RELLIS_LICENSE,
"archives": descriptors,
"identity_sha256": install_identity,
},
"alignment": {
"annotated_scan_count": len(split_pairs),
"split_counts": RELLIS_SPLIT_COUNTS,
"sequence_counts": dict(sorted(sequence_counts.items())),
"pose_counts": pose_counts,
"point_label_alignment": "complete",
"pose_index_coverage": "complete",
},
"frame": {
"frame_id": frame_id,
"split": "validation",
"source_point_count": frame.point_count,
"preview_point_count": int(preview["point_count"]),
"preview_sha256": _sha256_file(preview_path),
},
"coordinate_frame": {
"frame_id": "sensor/lidar/os1",
"x": "forward",
"y": "left",
"z": "up",
"transform_applied": False,
},
"next_action": "calibrate-on-train-then-qualify-validation",
}
_atomic_json(root / "state/rellis-3d-v1.1.json", manifest)
return manifest
def read_rellis_splits(dataset_root: Path) -> dict[str, tuple[tuple[str, str], ...]]:
"""Read the already admitted immutable split index."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise RellisAdmissionError("RELLIS split access requires the canonical worker D root")
split_path = root / RELLIS_ARCHIVE_ROOT / RELLIS_SPLIT_ARCHIVE
try:
with zipfile.ZipFile(split_path) as source:
return _read_splits(source, _member_index(source))
except (OSError, zipfile.BadZipFile) as exc:
raise RellisAdmissionError("RELLIS split archive cannot be indexed") from exc
def _archive_path(root: Path, value: Path | None, filename: str) -> Path:
return value.expanduser().absolute() if value is not None else root / filename
def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]:
members = source.infolist()
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
raise RellisAdmissionError("RELLIS archive entry count is outside the admitted range")
total = 0
result: dict[str, zipfile.ZipInfo] = {}
for member in members:
path = PurePosixPath(member.filename.replace("\\", "/"))
if path.is_absolute() or ".." in path.parts:
raise RellisAdmissionError("RELLIS archive contains an unsafe member path")
if (member.external_attr >> 16) & 0o170000 == 0o120000:
raise RellisAdmissionError("RELLIS archive contains a symbolic link")
total += member.file_size
if total > MAX_UNCOMPRESSED_BYTES:
raise RellisAdmissionError("RELLIS archive exceeds the uncompressed safety limit")
if member.is_dir():
continue
normalized = "/".join(part for part in path.parts if part not in {".", "Rellis-3D"})
if not normalized or normalized in result:
raise RellisAdmissionError("RELLIS archive member identity is ambiguous")
result[normalized] = member
return result
def _read_splits(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
) -> dict[str, tuple[tuple[str, str], ...]]:
result: dict[str, tuple[tuple[str, str], ...]] = {}
for split, expected in RELLIS_SPLIT_COUNTS.items():
filename = "pt_val.lst" if split == "validation" else f"pt_{split}.lst"
matches = [member for path, member in members.items() if path.endswith(filename)]
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
raise RellisAdmissionError(f"RELLIS {split} split index is unavailable")
try:
lines = source.read(matches[0]).decode("utf-8").splitlines()
except UnicodeDecodeError as exc:
raise RellisAdmissionError("RELLIS split index is not UTF-8") from exc
pairs: list[tuple[str, str]] = []
for raw in lines:
parts = raw.strip().replace("\\", "/").split()
if len(parts) != 2:
raise RellisAdmissionError("RELLIS split row is malformed")
point_path, label_path = parts
point_match = _FRAME_PATH.fullmatch(point_path)
label_match = _LABEL_PATH.fullmatch(label_path)
if (
point_match is None
or label_match is None
or point_match.group("sequence") != label_match.group("sequence")
or point_match.group("frame") != label_match.group("frame")
):
raise RellisAdmissionError("RELLIS split point and label identities differ")
pairs.append((point_path, label_path))
if len(pairs) != expected or len(set(pairs)) != expected:
raise RellisAdmissionError(f"RELLIS {split} split count differs")
result[split] = tuple(pairs)
if sum(map(len, result.values())) != len(
{pair for values in result.values() for pair in values}
):
raise RellisAdmissionError("RELLIS split indexes overlap")
return result
def _validate_member_alignment(
scans: dict[str, zipfile.ZipInfo],
labels: dict[str, zipfile.ZipInfo],
pairs: set[tuple[str, str]],
) -> None:
for point_path, label_path in pairs:
point = scans[point_path]
label = labels[label_path]
if (
point.file_size <= 0
or point.file_size % 16
or label.file_size % 4
or point.file_size // 16 != label.file_size // 4
):
raise RellisAdmissionError("RELLIS point and label member sizes are not aligned")
def _validate_poses(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
pairs: set[tuple[str, str]],
) -> dict[str, int]:
result: dict[str, int] = {}
required_by_sequence: dict[str, set[int]] = {
sequence: set() for sequence in RELLIS_SEQUENCE_IDS
}
for point_path, _ in pairs:
match = _FRAME_PATH.fullmatch(point_path)
assert match is not None
required_by_sequence[match.group("sequence")].add(int(match.group("frame")))
for sequence in RELLIS_SEQUENCE_IDS:
path = f"{sequence}/poses.txt"
matches = [member for name, member in members.items() if name.endswith(path)]
if len(matches) != 1 or matches[0].file_size > MAX_METADATA_MEMBER_BYTES:
raise RellisAdmissionError(f"RELLIS poses are unavailable for sequence {sequence}")
try:
rows = source.read(matches[0]).decode("utf-8").splitlines()
except UnicodeDecodeError as exc:
raise RellisAdmissionError("RELLIS pose file is not UTF-8") from exc
for row in rows:
values = row.split()
if len(values) != 12:
raise RellisAdmissionError("RELLIS pose row is not a 3x4 transform")
try:
tuple(float(value) for value in values)
except ValueError as exc:
raise RellisAdmissionError("RELLIS pose row is not numeric") from exc
if not rows or max(required_by_sequence[sequence], default=-1) >= len(rows):
raise RellisAdmissionError("RELLIS pose index does not cover every split frame")
result[sequence] = len(rows)
return result
def _qualified_frame_id(point_path: str) -> str:
match = _FRAME_PATH.fullmatch(point_path)
if match is None:
raise RellisAdmissionError("RELLIS frame path is incompatible")
return f"{match.group('sequence')}-{match.group('frame')}"
def _is_worker_dataset_root(root: Path) -> bool:
return str(root).replace("\\", "/").rstrip("/").lower() == (
"/mnt/d/ndc_missioncore/datasets"
)
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
try:
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
digest.update(chunk)
except OSError as exc:
raise RellisAdmissionError("RELLIS artifact cannot be hashed") from exc
return digest.hexdigest()
def _atomic_json(path: Path, document: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with tempfile.NamedTemporaryFile(
mode="w",
dir=path.parent,
encoding="utf-8",
delete=False,
) as temporary:
temporary_path = Path(temporary.name)
json.dump(document, temporary, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
temporary.write("\n")
temporary.flush()
os.fsync(temporary.fileno())
os.replace(temporary_path, path)
+104
View File
@@ -0,0 +1,104 @@
"""Train-calibrated, algorithm-scoped RELLIS OS1 Patchwork++ profile."""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Any, Final
RELLIS_PATCHWORK_PROFILE_SCHEMA: Final = "missioncore.rellis-patchwork-profile/v1"
RELLIS_URDF_URL: Final = (
"https://raw.githubusercontent.com/unmannedlab/RELLIS-3D/"
"c17a118fcaed1559f03cc32cc3a91dedc557f8b8/"
"catkin_ws/src/platform_description/urdf/warthog.urdf.xacro"
)
RELLIS_URDF_SHA256: Final = (
"bcb66a05b8818f06479bb4a17ce93e9d66cb8fdaab726c437596807288116ac3"
)
@dataclass(frozen=True, slots=True)
class RellisPatchworkProfile:
"""Frozen provider profile derived without looking at validation labels."""
patchwork_sensor_height_proxy_m: float
calibration_identity_sha256: str
calibration_frame_count: int
calibration_median_absolute_deviation_m: float
patchwork_minimum_range_m: float = 2.7
patchwork_maximum_range_m: float = 80.0
profile_id: str = "rellis-warthog-os1-patchwork/v1"
def __post_init__(self) -> None:
values = (
self.patchwork_sensor_height_proxy_m,
self.calibration_median_absolute_deviation_m,
self.patchwork_minimum_range_m,
self.patchwork_maximum_range_m,
)
if (
not self.profile_id
or not all(math.isfinite(value) for value in values)
or not 0.5 <= self.patchwork_sensor_height_proxy_m <= 3.0
or not 5 <= self.calibration_frame_count <= 256
or self.calibration_median_absolute_deviation_m < 0
or not 0 < self.patchwork_minimum_range_m < self.patchwork_maximum_range_m
or len(self.calibration_identity_sha256) != 64
or any(
character not in "0123456789abcdef"
for character in self.calibration_identity_sha256
)
):
raise ValueError("RELLIS Patchwork++ profile is invalid")
def to_dict(self) -> dict[str, Any]:
return {
"schema_version": RELLIS_PATCHWORK_PROFILE_SCHEMA,
"profile_id": self.profile_id,
"source_id": "rellis-3d/v1.1",
"representation": "native-scan",
"sensor_frame": {
"frame_id": "sensor/lidar/os1",
"handedness": "right",
"x": "forward",
"y": "left",
"z": "up",
"one_revolution": True,
},
"height": {
"sensor_above_ground_m": self.patchwork_sensor_height_proxy_m,
"evidence": "train-split-ground-label-calibration",
"calibration_frame_count": self.calibration_frame_count,
"median_absolute_deviation_m": (
self.calibration_median_absolute_deviation_m
),
"calibration_identity_sha256": self.calibration_identity_sha256,
"validation_labels_used": False,
},
"patchworkpp": {
"provider_id": "patchworkpp/v1.4.1",
"minimum_range_m": self.patchwork_minimum_range_m,
"maximum_range_m": self.patchwork_maximum_range_m,
"enable_rnr": True,
"enable_rvpf": True,
"enable_tgr": True,
},
"evidence": {
"vehicle_urdf": {
"url": RELLIS_URDF_URL,
"sha256": RELLIS_URDF_SHA256,
"use": "topology-cross-check-only",
"exact_physical_height_claimed": False,
}
},
"scope": {
"patchworkpp_eligible": True,
"normalized_scan_produced": False,
"complete_vehicle_transform_known": False,
"deskew_claimed": False,
},
"authority": {
"qualification_only": True,
"navigation_or_safety_accepted": False,
},
}
File diff suppressed because it is too large Load Diff
+3
View File
@@ -429,6 +429,9 @@ app.include_router(
dataset_rellis_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-preview.json"
),
dataset_rellis_admission_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "rellis-admission.json"
),
dataset_ground_preview_provider=lambda: (
REPOSITORY_ROOT / ".runtime" / "dataset-gateway" / "ground-comparison.json"
),
+2
View File
@@ -64,6 +64,7 @@ def build_lidar_router(
dataset_admission_provider: DatasetArtifactProvider = configured_dataset_admission_manifest,
dataset_preview_provider: DatasetArtifactProvider = configured_dataset_preview,
dataset_rellis_preview_provider: DatasetArtifactProvider = lambda: None,
dataset_rellis_admission_provider: DatasetArtifactProvider = lambda: None,
dataset_ground_preview_provider: DatasetArtifactProvider = configured_dataset_ground_preview,
) -> APIRouter:
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
@@ -73,6 +74,7 @@ def build_lidar_router(
return dataset_gateway_catalog(
admission_manifest_path=dataset_admission_provider(),
rellis_preview_path=dataset_rellis_preview_provider(),
rellis_admission_path=dataset_rellis_admission_provider(),
)
@router.get("/dataset-gateway/preview")
+46 -11
View File
@@ -41,17 +41,21 @@ DETAIL_SCHEMA: Final = "missioncore.polygon-run-detail/v1"
GROUND_QUALIFICATION_SCHEMA: Final = "missioncore.polygon-ground-qualification/v1"
GROUND_FAILURE_PREVIEW_SCHEMA: Final = "missioncore.polygon-ground-failure-preview/v1"
GROUND_REPORT_ARTIFACT_KIND: Final = "goose-ground-qualification-report"
RELLIS_REPORT_ARTIFACT_KIND: Final = "rellis-ground-qualification-report"
GROUND_FAILURE_ARTIFACT_KIND: Final = "goose-ground-qualification-failure-preview"
GROUND_REPORT_SCHEMA: Final = "missioncore.goose-ground-qualification-report/v1"
RELLIS_REPORT_SCHEMA: Final = "missioncore.rellis-ground-qualification-report/v1"
GROUND_FAILURE_SCHEMA: Final = "missioncore.goose-ground-qualification-failure-preview/v1"
GROUND_REVIEW_SCHEMA: Final = "missioncore.polygon-ground-review/v2"
GROUND_REVIEW_FRAME_SCHEMA: Final = "missioncore.polygon-ground-review-frame/v1"
GOOSE_REVIEW_PACK_SCHEMA: Final = "missioncore.goose-ground-review-pack/v1"
GOOSE_REVIEW_FRAME_SCHEMA: Final = "missioncore.goose-ground-review-frame/v1"
RELLIS_REVIEW_PACK_SCHEMA: Final = "missioncore.rellis-ground-review-pack/v1"
RELLIS_REVIEW_FRAME_SCHEMA: Final = "missioncore.rellis-ground-review-frame/v1"
MAX_QUALIFICATION_ARTIFACT_BYTES: Final = 32 * 1024**2
MAX_REVIEW_MANIFEST_BYTES: Final = 8 * 1024**2
MAX_REVIEW_FRAME_BYTES: Final = 2 * 1024**2
MAX_REVIEW_FRAMES: Final = 2_000
MAX_REVIEW_FRAMES: Final = 3_000
MAX_REVIEW_POINTS: Final = 20_000
COMMIT_PATTERN: Final = re.compile(r"^[a-f0-9]{40}$")
GOOSE_FRAME_ID_PATTERN: Final = re.compile(
@@ -201,8 +205,8 @@ def build_polygon_router(
report = _read_registered_json(
store,
run,
artifact_kind=GROUND_REPORT_ARTIFACT_KIND,
expected_schema=GROUND_REPORT_SCHEMA,
artifact_kind=(GROUND_REPORT_ARTIFACT_KIND, RELLIS_REPORT_ARTIFACT_KIND),
expected_schema=(GROUND_REPORT_SCHEMA, RELLIS_REPORT_SCHEMA),
)
required = (
"identity_sha256",
@@ -305,7 +309,7 @@ def build_polygon_router(
"ground_iou_delta",
)
},
**_goose_frame_identity(frame["frame_id"]),
**_dataset_frame_identity(frame),
}
for frame in manifest["frames"]
],
@@ -348,7 +352,11 @@ def build_polygon_router(
) from exc
point_count = int(frame["point_count"])
if (
schema != GOOSE_REVIEW_FRAME_SCHEMA
schema
not in {
GOOSE_REVIEW_FRAME_SCHEMA,
RELLIS_REVIEW_FRAME_SCHEMA,
}
or stored_frame_id != frame_id
or source_point_count != frame["source_point_count"]
or xyz_cm.dtype != np.dtype("<i2")
@@ -562,11 +570,17 @@ def _read_registered_json(
store: QualificationRunStore,
run: QualificationRun,
*,
artifact_kind: str,
expected_schema: str,
artifact_kind: str | tuple[str, ...],
expected_schema: str | tuple[str, ...],
frame_id: str | None = None,
) -> dict[str, Any]:
candidates = [artifact for artifact in run.artifacts if artifact.kind == artifact_kind]
artifact_kinds = (artifact_kind,) if isinstance(artifact_kind, str) else artifact_kind
expected_schemas = (
(expected_schema,) if isinstance(expected_schema, str) else expected_schema
)
candidates = [
artifact for artifact in run.artifacts if artifact.kind in artifact_kinds
]
if frame_id is not None:
candidates = [
artifact for artifact in candidates if Path(artifact.relative_path).stem == frame_id
@@ -611,7 +625,7 @@ def _read_registered_json(
status_code=500,
detail="Квалификационный артефакт не является допустимым JSON.",
) from exc
if not isinstance(value, dict) or value.get("schema_version") != expected_schema:
if not isinstance(value, dict) or value.get("schema_version") not in expected_schemas:
raise HTTPException(
status_code=500,
detail="Квалификационный артефакт имеет неизвестную схему.",
@@ -676,7 +690,8 @@ def _read_ground_review_manifest(
safety = manifest.get("safety") if isinstance(manifest, dict) else None
if (
not isinstance(manifest, dict)
or manifest.get("schema_version") != GOOSE_REVIEW_PACK_SCHEMA
or manifest.get("schema_version")
not in {GOOSE_REVIEW_PACK_SCHEMA, RELLIS_REVIEW_PACK_SCHEMA}
or manifest.get("source_run_id") != run_id
or not isinstance(manifest.get("identity_sha256"), str)
or not re.fullmatch(r"[a-f0-9]{64}", manifest["identity_sha256"])
@@ -725,7 +740,27 @@ def _read_ground_review_manifest(
return pack_root, manifest
def _goose_frame_identity(frame_id: str) -> dict[str, str | int]:
def _dataset_frame_identity(frame: dict[str, Any]) -> dict[str, str | int | None]:
frame_id = frame["frame_id"]
explicit_sequence = frame.get("dataset_sequence_id")
explicit_number = frame.get("dataset_frame_number")
if explicit_sequence is not None or explicit_number is not None:
if (
not isinstance(explicit_sequence, str)
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,95}", explicit_sequence)
is None
or not isinstance(explicit_number, int)
or explicit_number < 0
):
raise HTTPException(
status_code=500,
detail="Идентичность dataset-кадра нарушает контракт.",
)
return {
"dataset_sequence_id": explicit_sequence,
"dataset_frame_number": explicit_number,
"sensor_timestamp_ns": None,
}
match = GOOSE_FRAME_ID_PATTERN.fullmatch(frame_id)
if match is None:
raise HTTPException(