SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): add PointPillars transfer gate

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-31 10:48:27 +03:00
parent 29fee5c0ac
commit a9a5ca5a26
23 changed files with 5407 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/datasets/__init__.py
+14
View File
@@ -38,6 +38,14 @@ from k1link.datasets.goose_qualification import (
GroundAcceptancePolicy,
qualify_goose_ground,
)
from k1link.datasets.kitti_3d_admission import (
KITTI_3D_ADMISSION_SCHEMA,
KITTI_3D_SOURCE_ID,
Kitti3DAdmissionError,
admit_kitti_3d_object_release,
read_kitti_3d_admission,
read_kitti_standard_splits,
)
from k1link.datasets.rellis_admission import (
RELLIS_ADMISSION_SCHEMA,
RellisAdmissionError,
@@ -79,6 +87,8 @@ __all__ = [
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
"KITTI_3D_ADMISSION_SCHEMA",
"KITTI_3D_SOURCE_ID",
"RELLIS_CLASSES",
"RELLIS_ADMISSION_SCHEMA",
"RELLIS_GROUND_POLICY_SCHEMA",
@@ -91,6 +101,7 @@ __all__ = [
"RellisAdmissionError",
"RellisPatchworkProfile",
"RellisSmokeError",
"Kitti3DAdmissionError",
"GoosePatchworkProfile",
"GroundAcceptancePolicy",
"DegradationProfile",
@@ -98,6 +109,7 @@ __all__ = [
"benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground",
"admit_rellis_release",
"admit_kitti_3d_object_release",
"build_rellis_official_smoke_preview",
"calibrate_rellis_sensor_height",
"configured_dataset_admission_manifest",
@@ -108,6 +120,8 @@ __all__ = [
"read_dataset_admission_manifest",
"read_dataset_ground_preview",
"read_dataset_native_scan_preview",
"read_kitti_3d_admission",
"read_kitti_standard_splits",
"read_semantic_kitti_frame",
"rellis_native_scan_preview",
"qualify_rellis_ground",
src/k1link/datasets/cli.py
+21
View File
@@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
)
from k1link.datasets.goose_qualification import qualify_goose_ground
from k1link.datasets.goose_review import build_goose_ground_review_pack
from k1link.datasets.kitti_3d_admission import (
Kitti3DAdmissionError,
admit_kitti_3d_object_release,
)
from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release
from k1link.datasets.rellis_qualification import qualify_rellis_ground
from k1link.datasets.rellis_smoke import (
@@ -28,6 +32,23 @@ app = typer.Typer(
)
@app.command("admit-kitti-3d-object")
def admit_kitti_3d_object_command(
dataset_root: Annotated[
Path,
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
],
) -> None:
"""Verify the archive-only KITTI 3D release and standard validation split."""
try:
manifest = admit_kitti_3d_object_release(dataset_root)
except Kitti3DAdmissionError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
@app.command("admit-goose-validation")
def admit_goose_validation_command(
dataset_root: Annotated[
src/k1link/datasets/kitti_3d_admission.py
+552
View File
@@ -0,0 +1,552 @@
"""Fail-closed admission of the KITTI 3D object development release.
The release is admitted as independent oriented-3D-box truth for the L3
PointPillars baseline. Source archives stay on Worker 006. This module does
not extract data, install a model, run inference, or authorize a K1 quality
claim.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
import re
import tempfile
import zipfile
from collections import Counter
from contextlib import suppress
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Final
KITTI_3D_ADMISSION_SCHEMA: Final = "missioncore.kitti-3d-object-admission/v1"
KITTI_3D_SOURCE_ID: Final = "kitti-3d-object/v2017"
KITTI_3D_RELEASE_ROOT: Final = "kitti-3d-object/v2017"
KITTI_3D_LICENSE: Final = "CC-BY-NC-SA-3.0"
KITTI_3D_LICENSE_URL: Final = "https://www.cvlibs.net/datasets/kitti/index.php"
KITTI_3D_BENCHMARK_URL: Final = (
"https://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d"
)
KITTI_VELODYNE_ARCHIVE: Final = "data_object_velodyne.zip"
KITTI_LABEL_ARCHIVE: Final = "data_object_label_2.zip"
KITTI_CALIB_ARCHIVE: Final = "data_object_calib.zip"
KITTI_ARCHIVE_URLS: Final[dict[str, str]] = {
KITTI_VELODYNE_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_velodyne.zip"
),
KITTI_LABEL_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_label_2.zip"
),
KITTI_CALIB_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_calib.zip"
),
}
KITTI_ARCHIVE_BYTES: Final[dict[str, int]] = {
KITTI_VELODYNE_ARCHIVE: 28_750_710_812,
KITTI_LABEL_ARCHIVE: 5_601_213,
KITTI_CALIB_ARCHIVE: 26_854_811,
}
KITTI_TRAINING_FRAMES: Final = 7_481
KITTI_TEST_FRAMES: Final = 7_518
KITTI_TARGET_CLASSES: Final = ("Car", "Pedestrian", "Cyclist")
KITTI_STANDARD_SPLIT_COMMIT: Final = (
"233f849829b6ac19afb8af8837a0246890908755"
)
KITTI_STANDARD_SPLIT_URL: Final = (
"https://github.com/open-mmlab/OpenPCDet/tree/"
f"{KITTI_STANDARD_SPLIT_COMMIT}/data/kitti/ImageSets"
)
KITTI_SPLIT_FILES: Final[dict[str, str]] = {
"train": "train.txt",
"validation": "val.txt",
}
KITTI_SPLIT_COUNTS: Final[dict[str, int]] = {
"train": 3_712,
"validation": 3_769,
}
KITTI_SPLIT_SHA256: Final[dict[str, str]] = {
"train": "b6417a1d9b18c8fdb085128e633d28ff321b7674a6d1b3841b8f43d865b281cb",
"validation": (
"657ac4bcc1e156e5b106a4ca18e1f88e012787ea1d2b5d0adeea97fee903fa86"
),
}
MAX_ARCHIVE_ENTRIES: Final = 40_000
MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3
MAX_LABEL_MEMBER_BYTES: Final = 8 * 1024**2
_FRAME_ID = re.compile(r"^[0-9]{6}$")
_VELODYNE_MEMBER = re.compile(
r"^(?P<split>training|testing)/velodyne/(?P<frame>[0-9]{6})\.bin$"
)
_LABEL_MEMBER = re.compile(r"^training/label_2/(?P<frame>[0-9]{6})\.txt$")
_CALIB_MEMBER = re.compile(
r"^(?P<split>training|testing)/calib/(?P<frame>[0-9]{6})\.txt$"
)
_CALIB_KEYS: Final = {
"P0",
"P1",
"P2",
"P3",
"R0_rect",
"Tr_velo_to_cam",
"Tr_imu_to_velo",
}
class Kitti3DAdmissionError(RuntimeError):
"""The KITTI 3D development release violates its pinned contract."""
def admit_kitti_3d_object_release(
dataset_root: Path,
*,
velodyne_archive: Path | None = None,
label_archive: Path | None = None,
calib_archive: Path | None = None,
train_split: Path | None = None,
validation_split: Path | None = None,
) -> dict[str, Any]:
"""Verify the archive-only KITTI release and publish a path-free state."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise Kitti3DAdmissionError(
"KITTI admission requires the canonical Worker 006 D dataset root"
)
archive_root = root / KITTI_3D_RELEASE_ROOT / "archives"
split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / (
f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}"
)
archive_paths = {
KITTI_VELODYNE_ARCHIVE: _resolved_input(
archive_root, velodyne_archive, KITTI_VELODYNE_ARCHIVE
),
KITTI_LABEL_ARCHIVE: _resolved_input(
archive_root, label_archive, KITTI_LABEL_ARCHIVE
),
KITTI_CALIB_ARCHIVE: _resolved_input(
archive_root, calib_archive, KITTI_CALIB_ARCHIVE
),
}
split_paths = {
"train": _resolved_input(
split_root, train_split, KITTI_SPLIT_FILES["train"]
),
"validation": _resolved_input(
split_root, validation_split, KITTI_SPLIT_FILES["validation"]
),
}
if any(not path.is_file() for path in (*archive_paths.values(), *split_paths.values())):
raise Kitti3DAdmissionError("one or more pinned KITTI artifacts are unavailable")
archives: dict[str, dict[str, Any]] = {}
for filename, path in archive_paths.items():
size_bytes = path.stat().st_size
if size_bytes != KITTI_ARCHIVE_BYTES[filename]:
raise Kitti3DAdmissionError(
f"{filename} size differs from the pinned KITTI release"
)
archives[filename] = {
"filename": filename,
"source_url": KITTI_ARCHIVE_URLS[filename],
"size_bytes": size_bytes,
"sha256": _sha256_file(path),
"vendor_checksum_available": False,
}
splits = _read_standard_splits(split_paths)
try:
with (
zipfile.ZipFile(archive_paths[KITTI_VELODYNE_ARCHIVE]) as points_zip,
zipfile.ZipFile(archive_paths[KITTI_LABEL_ARCHIVE]) as labels_zip,
zipfile.ZipFile(archive_paths[KITTI_CALIB_ARCHIVE]) as calib_zip,
):
point_members = _member_index(points_zip)
label_members = _member_index(labels_zip)
calib_members = _member_index(calib_zip)
training_points, testing_points = _validate_velodyne(point_members)
training_labels, target_counts = _validate_labels(
labels_zip, label_members
)
training_calib, testing_calib = _validate_calibrations(
calib_zip, calib_members
)
except (OSError, KeyError, UnicodeDecodeError, zipfile.BadZipFile) as exc:
raise Kitti3DAdmissionError("KITTI archives could not be verified") from exc
training_ids = set(training_points)
if (
set(training_labels) != training_ids
or set(training_calib) != training_ids
or set(testing_points) != set(testing_calib)
):
raise Kitti3DAdmissionError("KITTI point, label, and calibration indices diverge")
split_union = set(splits["train"]) | set(splits["validation"])
if (
set(splits["train"]).intersection(splits["validation"])
or split_union != training_ids
):
raise Kitti3DAdmissionError(
"OpenPCDet train/validation split is overlapping or incomplete"
)
validation_target_counts = _target_counts_for_frames(
archive_paths[KITTI_LABEL_ARCHIVE],
set(splits["validation"]),
)
if any(validation_target_counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES):
raise Kitti3DAdmissionError("KITTI validation split lacks a target class")
identity = {
"source_id": KITTI_3D_SOURCE_ID,
"archives": archives,
"split_source": {
"repository_commit": KITTI_STANDARD_SPLIT_COMMIT,
"source_url": KITTI_STANDARD_SPLIT_URL,
"sha256": KITTI_SPLIT_SHA256,
},
"license": {
"spdx": KITTI_3D_LICENSE,
"source_url": KITTI_3D_LICENSE_URL,
"use_scope": "academic-non-commercial",
},
"representation": {
"point_fields": ["x", "y", "z", "intensity"],
"ground_truth": "oriented-3d-boxes",
"box_coordinate_frame": "camera-rectified",
"calibration_to_sensor_frame_present": True,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
manifest = {
"schema_version": KITTI_3D_ADMISSION_SCHEMA,
"source_id": KITTI_3D_SOURCE_ID,
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"status": "archive-ready",
"release_identity_sha256": identity_sha256,
"storage": {
"policy": "worker-d-only",
"admitted": True,
"canonical_root": True,
"path_exposed": False,
"source_archives_extracted": False,
},
"identity": identity,
"alignment": {
"training_frame_count": len(training_points),
"test_frame_count": len(testing_points),
"training_label_count": len(training_labels),
"training_calibration_count": len(training_calib),
"test_calibration_count": len(testing_calib),
"split_counts": KITTI_SPLIT_COUNTS,
"split_union_complete": True,
"split_overlap_count": 0,
"all_target_box_counts": dict(sorted(target_counts.items())),
"validation_target_box_counts": dict(
sorted(validation_target_counts.items())
),
},
"benchmark_contract": {
"independent_ground_truth": True,
"annotations": ["oriented-3d-boxes"],
"point_fields": ["x", "y", "z", "intensity"],
"eligible_split": "validation",
"target_classes": list(KITTI_TARGET_CLASSES),
"official_test_submission_authorized": False,
"retuning_on_validation_allowed": False,
"k1_quality_claim_authorized": False,
},
"next_action": "promote-staged-engine-then-stream-standard-validation",
}
_atomic_json(root / "state/kitti-3d-object-v2017.json", manifest)
return manifest
def read_kitti_3d_admission(dataset_root: Path) -> dict[str, Any]:
"""Read the current path-free state and validate its content identity."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise Kitti3DAdmissionError(
"KITTI admission requires the canonical Worker 006 D dataset root"
)
path = root / "state/kitti-3d-object-v2017.json"
try:
manifest = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise Kitti3DAdmissionError("KITTI admission state is unavailable") from exc
if not isinstance(manifest, dict):
raise Kitti3DAdmissionError("KITTI admission state is not an object")
identity = manifest.get("identity")
if (
manifest.get("schema_version") != KITTI_3D_ADMISSION_SCHEMA
or manifest.get("source_id") != KITTI_3D_SOURCE_ID
or manifest.get("status") != "archive-ready"
or not isinstance(identity, dict)
or manifest.get("release_identity_sha256")
!= hashlib.sha256(_canonical_json(identity)).hexdigest()
):
raise Kitti3DAdmissionError("KITTI admission state identity is invalid")
return manifest
def read_kitti_standard_splits(
dataset_root: Path,
) -> dict[str, tuple[str, ...]]:
"""Read the pinned OpenPCDet split files after validating current state."""
root = dataset_root.expanduser().absolute()
read_kitti_3d_admission(root)
split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / (
f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}"
)
return _read_standard_splits(
{
split_name: split_root / filename
for split_name, filename in KITTI_SPLIT_FILES.items()
}
)
def _resolved_input(root: Path, provided: Path | None, filename: str) -> Path:
return provided.expanduser().absolute() if provided is not None else root / filename
def _is_worker_dataset_root(root: Path) -> bool:
normalized = str(root).replace("\\", "/").rstrip("/").lower()
return normalized == "/mnt/d/ndc_missioncore/datasets"
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
try:
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
digest.update(chunk)
except OSError as exc:
raise Kitti3DAdmissionError("KITTI artifact cannot be hashed") from exc
return digest.hexdigest()
def _read_standard_splits(paths: dict[str, Path]) -> dict[str, tuple[str, ...]]:
result: dict[str, tuple[str, ...]] = {}
for split_name, path in paths.items():
if _sha256_file(path) != KITTI_SPLIT_SHA256[split_name]:
raise Kitti3DAdmissionError(
f"KITTI {split_name} split differs from the pinned OpenPCDet commit"
)
try:
rows = tuple(
row.strip()
for row in path.read_text(encoding="ascii").splitlines()
if row.strip()
)
except (OSError, UnicodeDecodeError) as exc:
raise Kitti3DAdmissionError("KITTI split cannot be read") from exc
if (
len(rows) != KITTI_SPLIT_COUNTS[split_name]
or len(set(rows)) != len(rows)
or any(_FRAME_ID.fullmatch(row) is None for row in rows)
):
raise Kitti3DAdmissionError(f"KITTI {split_name} split is invalid")
result[split_name] = rows
return result
def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]:
members = source.infolist()
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
raise Kitti3DAdmissionError("KITTI archive entry count is invalid")
total_uncompressed = 0
indexed: dict[str, zipfile.ZipInfo] = {}
for member in members:
path = PurePosixPath(member.filename)
if (
path.is_absolute()
or ".." in path.parts
or "\\" in member.filename
or member.file_size < 0
or member.compress_size < 0
):
raise Kitti3DAdmissionError("KITTI archive contains an unsafe member")
total_uncompressed += member.file_size
if total_uncompressed > MAX_UNCOMPRESSED_BYTES:
raise Kitti3DAdmissionError("KITTI archive expands beyond the admitted limit")
if member.is_dir():
continue
normalized = path.as_posix()
if normalized in indexed:
raise Kitti3DAdmissionError("KITTI archive contains duplicate members")
indexed[normalized] = member
return indexed
def _validate_velodyne(
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]:
indexed: dict[str, dict[str, zipfile.ZipInfo]] = {
"training": {},
"testing": {},
}
for path, member in members.items():
match = _VELODYNE_MEMBER.fullmatch(path)
if match is None:
continue
if member.file_size <= 0 or member.file_size % 16:
raise Kitti3DAdmissionError("KITTI Velodyne frame is not packed XYZI")
indexed[match.group("split")][match.group("frame")] = member
if (
len(indexed["training"]) != KITTI_TRAINING_FRAMES
or len(indexed["testing"]) != KITTI_TEST_FRAMES
):
raise Kitti3DAdmissionError("KITTI Velodyne frame count is invalid")
return indexed["training"], indexed["testing"]
def _validate_labels(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], Counter[str]]:
indexed: dict[str, zipfile.ZipInfo] = {}
counts: Counter[str] = Counter()
for path, member in members.items():
match = _LABEL_MEMBER.fullmatch(path)
if match is None:
continue
if member.file_size > MAX_LABEL_MEMBER_BYTES:
raise Kitti3DAdmissionError("KITTI label member is unexpectedly large")
frame_id = match.group("frame")
indexed[frame_id] = member
counts.update(_parse_label_member(source.read(member)))
if len(indexed) != KITTI_TRAINING_FRAMES:
raise Kitti3DAdmissionError("KITTI label frame count is invalid")
if any(counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES):
raise Kitti3DAdmissionError("KITTI release lacks a target 3D box class")
return indexed, counts
def _parse_label_member(payload: bytes) -> Counter[str]:
try:
text = payload.decode("ascii")
except UnicodeDecodeError as exc:
raise Kitti3DAdmissionError("KITTI label member is not ASCII") from exc
counts: Counter[str] = Counter()
for raw_line in text.splitlines():
fields = raw_line.split()
if not fields:
continue
if len(fields) != 15:
raise Kitti3DAdmissionError("KITTI label row does not have 15 fields")
class_name = fields[0]
try:
values = [float(value) for value in fields[1:]]
except ValueError as exc:
raise Kitti3DAdmissionError("KITTI label row contains invalid numbers") from exc
if not all(math.isfinite(value) for value in values):
raise Kitti3DAdmissionError("KITTI label row contains non-finite numbers")
if class_name in KITTI_TARGET_CLASSES:
height, width, length = values[7:10]
if height <= 0 or width <= 0 or length <= 0:
raise Kitti3DAdmissionError("KITTI target box has invalid dimensions")
counts[class_name] += 1
return counts
def _validate_calibrations(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]:
indexed: dict[str, dict[str, zipfile.ZipInfo]] = {
"training": {},
"testing": {},
}
for path, member in members.items():
match = _CALIB_MEMBER.fullmatch(path)
if match is None:
continue
payload = source.read(member)
try:
lines = payload.decode("ascii").splitlines()
except UnicodeDecodeError as exc:
raise Kitti3DAdmissionError("KITTI calibration is not ASCII") from exc
keys: set[str] = set()
for line in lines:
if not line.strip():
continue
key, separator, raw_values = line.partition(":")
if not separator:
raise Kitti3DAdmissionError("KITTI calibration row is invalid")
try:
values = [float(value) for value in raw_values.split()]
except ValueError as exc:
raise Kitti3DAdmissionError(
"KITTI calibration contains invalid numbers"
) from exc
if not values or not all(math.isfinite(value) for value in values):
raise Kitti3DAdmissionError(
"KITTI calibration contains non-finite numbers"
)
keys.add(key)
if not _CALIB_KEYS.issubset(keys):
raise Kitti3DAdmissionError("KITTI calibration lacks required transforms")
indexed[match.group("split")][match.group("frame")] = member
if (
len(indexed["training"]) != KITTI_TRAINING_FRAMES
or len(indexed["testing"]) != KITTI_TEST_FRAMES
):
raise Kitti3DAdmissionError("KITTI calibration frame count is invalid")
return indexed["training"], indexed["testing"]
def _target_counts_for_frames(
labels_archive: Path,
frame_ids: set[str],
) -> Counter[str]:
counts: Counter[str] = Counter()
try:
with zipfile.ZipFile(labels_archive) as source:
members = _member_index(source)
for frame_id in sorted(frame_ids):
path = f"training/label_2/{frame_id}.txt"
member = members.get(path)
if member is None:
raise Kitti3DAdmissionError(
"KITTI validation split references a missing label"
)
counts.update(_parse_label_member(source.read(member)))
except (OSError, zipfile.BadZipFile) as exc:
raise Kitti3DAdmissionError(
"KITTI validation labels could not be verified"
) from exc
return counts
def _canonical_json(payload: Any) -> bytes:
return json.dumps(
payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _atomic_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(
dir=path.parent,
prefix=f".{path.name}.",
suffix=".tmp",
)
try:
with os.fdopen(descriptor, "wb") as target:
target.write(_canonical_json(payload) + b"\n")
target.flush()
os.fsync(target.fileno())
os.replace(temporary, path)
except BaseException:
with suppress(OSError):
os.unlink(temporary)
raise