feat(lidar): add dataset gateway boundary
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,17 @@
|
||||
"""Vendor-neutral dataset ingress for perception qualification."""
|
||||
|
||||
from k1link.datasets.gateway import (
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||
DatasetFrameError,
|
||||
DatasetPointFrame,
|
||||
dataset_gateway_catalog,
|
||||
read_semantic_kitti_frame,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DATASET_GATEWAY_CATALOG_SCHEMA",
|
||||
"DatasetFrameError",
|
||||
"DatasetPointFrame",
|
||||
"dataset_gateway_catalog",
|
||||
"read_semantic_kitti_frame",
|
||||
]
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Dataset-first LiDAR ingress with explicit representation boundaries.
|
||||
|
||||
The gateway never treats a registered map increment as a native sensor scan.
|
||||
It first preserves one source frame and its labels. Normalization and rolling
|
||||
map construction are separate, provenance-bearing products.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from typing import Final, Literal
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v1"
|
||||
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
|
||||
Representation = Literal["native-scan", "normalized-scan", "rolling-local-map"]
|
||||
|
||||
|
||||
class DatasetFrameError(ValueError):
|
||||
"""A dataset frame cannot satisfy its declared lossless source contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetPointFrame:
|
||||
"""One SemanticKITTI-compatible labeled LiDAR frame.
|
||||
|
||||
GOOSE publishes one LiDAR revolution per ``.bin`` frame. Coordinates and
|
||||
remission stay source-native here; no vehicle transform or accumulation is
|
||||
silently applied.
|
||||
"""
|
||||
|
||||
points_xyz_m: npt.NDArray[np.float32]
|
||||
remission: npt.NDArray[np.float32]
|
||||
semantic_labels: npt.NDArray[np.uint16]
|
||||
instance_labels: npt.NDArray[np.uint16]
|
||||
representation: Representation = "native-scan"
|
||||
|
||||
@property
|
||||
def point_count(self) -> int:
|
||||
return int(self.points_xyz_m.shape[0])
|
||||
|
||||
|
||||
def read_semantic_kitti_frame(
|
||||
point_path: Path,
|
||||
label_path: Path,
|
||||
*,
|
||||
maximum_points: int = 2_000_000,
|
||||
) -> DatasetPointFrame:
|
||||
"""Read one GOOSE/SemanticKITTI XYZI + packed-label frame losslessly."""
|
||||
|
||||
if maximum_points <= 0:
|
||||
raise DatasetFrameError("maximum_points must be positive")
|
||||
try:
|
||||
point_size = point_path.stat().st_size
|
||||
label_size = label_path.stat().st_size
|
||||
except OSError as exc:
|
||||
raise DatasetFrameError("dataset point or label file is unavailable") from exc
|
||||
if point_size == 0 or point_size % 16:
|
||||
raise DatasetFrameError("point frame must contain little-endian float32 XYZI tuples")
|
||||
if label_size % 4:
|
||||
raise DatasetFrameError("label frame must contain packed little-endian uint32 values")
|
||||
point_count = point_size // 16
|
||||
if point_count > maximum_points:
|
||||
raise DatasetFrameError("dataset point frame exceeds the configured safety limit")
|
||||
if label_size // 4 != point_count:
|
||||
raise DatasetFrameError("point and label counts do not match")
|
||||
|
||||
try:
|
||||
xyzi = np.fromfile(point_path, dtype="<f4").reshape((-1, 4))
|
||||
packed_labels = np.fromfile(label_path, dtype="<u4")
|
||||
except (OSError, ValueError) as exc:
|
||||
raise DatasetFrameError("dataset frame cannot be decoded") from exc
|
||||
if not np.isfinite(xyzi).all():
|
||||
raise DatasetFrameError("dataset point frame contains non-finite values")
|
||||
|
||||
points = np.ascontiguousarray(xyzi[:, :3], dtype=np.float32)
|
||||
remission = np.ascontiguousarray(xyzi[:, 3], dtype=np.float32)
|
||||
semantic = np.ascontiguousarray(packed_labels & np.uint32(0xFFFF), dtype=np.uint16)
|
||||
instance = np.ascontiguousarray(packed_labels >> np.uint32(16), dtype=np.uint16)
|
||||
for values in (points, remission, semantic, instance):
|
||||
values.setflags(write=False)
|
||||
return DatasetPointFrame(points, remission, semantic, instance)
|
||||
|
||||
|
||||
def _configured_dataset_root() -> Path | None:
|
||||
raw = os.environ.get(DATASET_ROOT_ENV, "").strip()
|
||||
return Path(raw).expanduser().absolute() if raw else None
|
||||
|
||||
|
||||
def _is_worker_d_storage(root: Path | None) -> bool:
|
||||
if root is None:
|
||||
return False
|
||||
raw = str(root)
|
||||
windows = PureWindowsPath(raw)
|
||||
if windows.drive.upper() == "D:":
|
||||
return True
|
||||
normalized = raw.replace("\\", "/").rstrip("/").lower()
|
||||
return normalized == "/mnt/d/ndc_missioncore/datasets" or normalized.startswith(
|
||||
"/mnt/d/ndc_missioncore/datasets/"
|
||||
)
|
||||
|
||||
|
||||
def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, object]:
|
||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||
|
||||
root = dataset_root if dataset_root is not None else _configured_dataset_root()
|
||||
storage_admitted = _is_worker_d_storage(root)
|
||||
return {
|
||||
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||
"access": "read-only",
|
||||
"storage": {
|
||||
"configured": root is not None,
|
||||
"required_windows_root": r"D:\NDC_MISSIONCORE\datasets",
|
||||
"required_wsl_root": "/mnt/d/NDC_MISSIONCORE/datasets",
|
||||
"admitted": storage_admitted,
|
||||
"status": "ready" if storage_admitted else "blocked-storage-policy",
|
||||
"path_exposed": False,
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"source_id": "goose-3d/v2025-08-22",
|
||||
"display_name": "GOOSE 3D",
|
||||
"role": "primary-offroad-semantic-baseline",
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"format": "semantickitti-xyzi-label",
|
||||
"frame_semantics": "one-lidar-revolution",
|
||||
"platforms": ["MuCAR-3", "ALICE", "Spot"],
|
||||
"annotations": ["semantic-point", "instance-point"],
|
||||
"superclasses": [
|
||||
"other",
|
||||
"artificial-structures",
|
||||
"artificial-ground",
|
||||
"natural-ground",
|
||||
"obstacle",
|
||||
"vehicle",
|
||||
"vegetation",
|
||||
"human",
|
||||
"sky",
|
||||
],
|
||||
"download": {
|
||||
"automatic": False,
|
||||
"reason": "operator-admitted-large-artifact-only",
|
||||
"training_archive_gb": 27.0,
|
||||
"validation_archive_gb": 3.3,
|
||||
"test_archive_gb": 3.3,
|
||||
},
|
||||
"admission": {
|
||||
"status": (
|
||||
"ready-for-download" if storage_admitted else "blocked-storage-policy"
|
||||
),
|
||||
"native_scan": "ready-after-download",
|
||||
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
|
||||
"rolling_local_map": "requires-pose-timing-and-map-policy",
|
||||
},
|
||||
}
|
||||
],
|
||||
"representations": [
|
||||
{
|
||||
"id": "native-scan",
|
||||
"title": "Одиночный исходный скан",
|
||||
"retains": ["xyz", "remission", "semantic-label", "instance-label"],
|
||||
"purpose": "dataset-ground-truth-and-sensor-domain",
|
||||
"accumulation": False,
|
||||
},
|
||||
{
|
||||
"id": "normalized-scan",
|
||||
"title": "Нормализованный sensor-frame скан",
|
||||
"retains": ["source-provenance", "point-alignment", "labels"],
|
||||
"purpose": "deskew-filter-inference-and-algorithm-comparison",
|
||||
"accumulation": False,
|
||||
},
|
||||
{
|
||||
"id": "rolling-local-map",
|
||||
"title": "Накопленная локальная карта",
|
||||
"retains": ["source-frame-ids", "pose-provenance", "age"],
|
||||
"purpose": "stable-operator-view-and-local-planning",
|
||||
"accumulation": True,
|
||||
},
|
||||
],
|
||||
"pipeline": [
|
||||
{
|
||||
"stage": "decode-and-calibrate",
|
||||
"requires": ["native-packets-or-source-frame", "calibration"],
|
||||
"produces": "native-scan",
|
||||
},
|
||||
{
|
||||
"stage": "deskew",
|
||||
"requires": ["per-point-time", "imu-or-odometry"],
|
||||
"produces": "normalized-scan",
|
||||
},
|
||||
{
|
||||
"stage": "bounded-cleanup",
|
||||
"requires": ["range-policy", "self-mask", "outlier-policy", "voxel-policy"],
|
||||
"produces": "normalized-scan",
|
||||
},
|
||||
{
|
||||
"stage": "ground-and-object-inference",
|
||||
"requires": ["normalized-scan", "declared-sensor-height-and-geometry"],
|
||||
"produces": "point-aligned-predictions",
|
||||
},
|
||||
{
|
||||
"stage": "pose-registration-and-rolling-map",
|
||||
"requires": ["normalized-scan", "pose", "ttl", "voxel-deduplication"],
|
||||
"produces": "rolling-local-map",
|
||||
},
|
||||
],
|
||||
"known_inputs": [
|
||||
{
|
||||
"source_id": "current-recorded-lidar/vendor-map",
|
||||
"representation": "vendor-mapped-increment",
|
||||
"native_scan": False,
|
||||
"per_point_time": False,
|
||||
"ring_or_line": False,
|
||||
"admitted_for_patchworkpp": False,
|
||||
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
|
||||
}
|
||||
],
|
||||
"next_action": (
|
||||
"download-goose-validation-to-d"
|
||||
if storage_admitted
|
||||
else "configure-dataset-root-on-worker-d"
|
||||
),
|
||||
}
|
||||
@@ -20,6 +20,7 @@ from k1link.compute import (
|
||||
lidar_pack_catalog_item,
|
||||
lidar_pack_detail,
|
||||
)
|
||||
from k1link.datasets import dataset_gateway_catalog
|
||||
|
||||
LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1"
|
||||
LIDAR_GROUND_CATALOG_SCHEMA: Final = "missioncore.lidar-ground-benchmark-catalog/v1"
|
||||
@@ -53,6 +54,10 @@ def build_lidar_router(
|
||||
) -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
|
||||
|
||||
@router.get("/dataset-gateway")
|
||||
def get_dataset_gateway() -> dict[str, object]:
|
||||
return dataset_gateway_catalog()
|
||||
|
||||
@router.get("/replay-packs")
|
||||
def list_lidar_replay_packs(
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
|
||||
Reference in New Issue
Block a user