407 lines
15 KiB
Python
407 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from fastapi import APIRouter
|
|
from fastapi.routing import APIRoute
|
|
|
|
from k1link.datasets import (
|
|
DatasetAdmissionError,
|
|
DatasetFrameError,
|
|
dataset_gateway_catalog,
|
|
goose_admission,
|
|
goose_benchmark,
|
|
read_dataset_admission_manifest,
|
|
read_dataset_ground_preview,
|
|
read_dataset_native_scan_preview,
|
|
read_semantic_kitti_frame,
|
|
)
|
|
from k1link.datasets.goose_admission import admit_goose_validation
|
|
from k1link.datasets.goose_benchmark import (
|
|
benchmark_goose_current_ground,
|
|
benchmark_goose_patchwork_ground,
|
|
)
|
|
from k1link.ground_segmentation import GroundSegmentation
|
|
from k1link.web.lidar_api import build_lidar_router
|
|
|
|
|
|
def _endpoint(router: APIRouter, path: str) -> object:
|
|
for route in router.routes:
|
|
if isinstance(route, APIRoute) and route.path == path and "GET" in route.methods:
|
|
return route.endpoint
|
|
raise AssertionError(f"GET {path} route is missing")
|
|
|
|
|
|
def test_goose_semantickitti_frame_retains_xyzi_and_packed_labels(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
points_path = tmp_path / "frame_vls128.bin"
|
|
labels_path = tmp_path / "frame_goose.label"
|
|
np.asarray(
|
|
[
|
|
[1.0, 2.0, 3.0, 0.25],
|
|
[-4.0, 5.0, 0.5, 0.75],
|
|
],
|
|
dtype="<f4",
|
|
).tofile(points_path)
|
|
np.asarray(
|
|
[
|
|
(12 << 16) | 3,
|
|
(99 << 16) | 42,
|
|
],
|
|
dtype="<u4",
|
|
).tofile(labels_path)
|
|
|
|
frame = read_semantic_kitti_frame(points_path, labels_path)
|
|
|
|
assert frame.representation == "native-scan"
|
|
assert frame.point_count == 2
|
|
assert frame.points_xyz_m.tolist() == [[1.0, 2.0, 3.0], [-4.0, 5.0, 0.5]]
|
|
assert frame.remission.tolist() == [0.25, 0.75]
|
|
assert frame.semantic_labels.tolist() == [3, 42]
|
|
assert frame.instance_labels.tolist() == [12, 99]
|
|
assert frame.points_xyz_m.flags.writeable is False
|
|
|
|
|
|
def test_dataset_frame_fails_closed_on_misaligned_or_nonfinite_input(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
points_path = tmp_path / "frame.bin"
|
|
labels_path = tmp_path / "frame.label"
|
|
np.asarray([[1.0, 2.0, 3.0, 0.25]], dtype="<f4").tofile(points_path)
|
|
np.asarray([1, 2], dtype="<u4").tofile(labels_path)
|
|
with pytest.raises(DatasetFrameError, match="counts do not match"):
|
|
read_semantic_kitti_frame(points_path, labels_path)
|
|
|
|
np.asarray([[np.nan, 2.0, 3.0, 0.25]], dtype="<f4").tofile(points_path)
|
|
np.asarray([1], dtype="<u4").tofile(labels_path)
|
|
with pytest.raises(DatasetFrameError, match="non-finite"):
|
|
read_semantic_kitti_frame(points_path, labels_path)
|
|
|
|
|
|
def test_dataset_gateway_enforces_d_storage_and_representation_boundaries(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
blocked = dataset_gateway_catalog(tmp_path / "datasets")
|
|
admitted = dataset_gateway_catalog(Path("/mnt/d/NDC_MISSIONCORE/datasets"))
|
|
|
|
assert blocked["storage"]["status"] == "blocked-storage-policy" # type: ignore[index]
|
|
assert admitted["storage"]["status"] == "ready" # type: ignore[index]
|
|
assert admitted["next_action"] == "download-goose-validation-to-d"
|
|
representations = admitted["representations"]
|
|
assert isinstance(representations, list)
|
|
assert [item["id"] for item in representations] == [ # type: ignore[index]
|
|
"native-scan",
|
|
"normalized-scan",
|
|
"rolling-local-map",
|
|
]
|
|
inputs = admitted["known_inputs"]
|
|
assert isinstance(inputs, list)
|
|
assert inputs[0]["admitted_for_patchworkpp"] is False # type: ignore[index]
|
|
|
|
|
|
def test_dataset_gateway_api_is_read_only_and_path_free(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setenv("MISSIONCORE_DATASET_ROOT", "/mnt/d/NDC_MISSIONCORE/datasets")
|
|
route = _endpoint(build_lidar_router(), "/api/v1/lidar/dataset-gateway")
|
|
response = route() # type: ignore[operator]
|
|
|
|
assert response["schema_version"] == "missioncore.dataset-gateway-catalog/v2"
|
|
assert response["access"] == "read-only"
|
|
assert response["storage"]["admitted"] is True
|
|
assert response["storage"]["path_exposed"] is False
|
|
assert "/mnt/d/NDC_MISSIONCORE/datasets" not in repr(response).replace(
|
|
response["storage"]["required_wsl_root"], # type: ignore[index]
|
|
"",
|
|
)
|
|
|
|
|
|
def _downloading_manifest() -> dict[str, object]:
|
|
return {
|
|
"schema_version": "missioncore.dataset-admission/v1",
|
|
"source_id": "goose-3d/v2025-08-22",
|
|
"observed_at_utc": "2026-07-25T10:00:00Z",
|
|
"status": "downloading",
|
|
"storage": {
|
|
"policy": "worker-d-only",
|
|
"admitted": True,
|
|
"canonical_root": True,
|
|
"path_exposed": False,
|
|
},
|
|
"archive": {
|
|
"filename": "goose_3d_val.zip",
|
|
"source_url": "https://goose-dataset.de/storage/goose_3d_val.zip",
|
|
"bytes_transferred": 512,
|
|
"total_bytes": 1024,
|
|
"size_bytes": None,
|
|
"sha256": None,
|
|
"integrity": "pending",
|
|
"vendor_checksum_available": False,
|
|
},
|
|
"license": {
|
|
"spdx": "CC-BY-SA-4.0",
|
|
"artifact_present": False,
|
|
},
|
|
"frame": None,
|
|
"next_action": "complete-download-and-admit",
|
|
}
|
|
|
|
|
|
def test_dataset_gateway_uses_worker_manifest_instead_of_static_download_state(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
manifest_path = tmp_path / "admission.json"
|
|
manifest_path.write_text(json.dumps(_downloading_manifest()), encoding="utf-8")
|
|
|
|
catalog = dataset_gateway_catalog(
|
|
tmp_path / "not-worker-d",
|
|
admission_manifest_path=manifest_path,
|
|
)
|
|
|
|
assert catalog["storage"]["admitted"] is True # type: ignore[index]
|
|
assert catalog["storage"]["attestation"] == "worker-manifest" # type: ignore[index]
|
|
source = catalog["sources"][0] # type: ignore[index]
|
|
assert source["admission"]["status"] == "downloading" # type: ignore[index]
|
|
assert source["admission"]["archive"]["bytes_transferred"] == 512 # type: ignore[index]
|
|
assert catalog["next_action"] == "complete-download-and-admit"
|
|
|
|
|
|
def test_dataset_admission_manifest_fails_closed_on_forged_storage(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
manifest = _downloading_manifest()
|
|
manifest["storage"]["path_exposed"] = True # type: ignore[index]
|
|
path = tmp_path / "admission.json"
|
|
path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
|
|
with pytest.raises(DatasetAdmissionError, match="storage admission"):
|
|
read_dataset_admission_manifest(path)
|
|
|
|
|
|
def test_goose_admission_extracts_one_aligned_frame_and_bounded_preview(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
root = tmp_path / "datasets"
|
|
archive = root / "goose-3d/v2025-08-22/archives/goose_3d_val.zip"
|
|
archive.parent.mkdir(parents=True)
|
|
points = np.asarray(
|
|
[
|
|
[1.0, 2.0, 3.0, 0.25],
|
|
[-4.0, 5.0, 0.5, 0.75],
|
|
[2.0, 1.0, -0.2, 0.5],
|
|
],
|
|
dtype="<f4",
|
|
)
|
|
packed_labels = np.asarray([23, 38, 31], dtype="<u4")
|
|
mapping = (
|
|
"class_name,label_key,has_instance,hex,challege_category_id,"
|
|
"challenge_category_name\n"
|
|
"asphalt,23,0,#ff2f80,2,artificial_ground\n"
|
|
"soil,31,0,#d16100,3,natural_ground\n"
|
|
"building,38,0,#013349,1,artificial_structures\n"
|
|
)
|
|
with zipfile.ZipFile(archive, "w") as target:
|
|
target.writestr(
|
|
"goose/velodyne/val/scene/2026_frame_vls128.bin",
|
|
points.tobytes(),
|
|
)
|
|
target.writestr(
|
|
"goose/labels/val/scene/2026_frame_goose.label",
|
|
packed_labels.tobytes(),
|
|
)
|
|
target.writestr("goose/goose_label_mapping.csv", mapping)
|
|
target.writestr(
|
|
"goose/LICENSE",
|
|
"Creative Commons Attribution-ShareAlike 4.0 International",
|
|
)
|
|
monkeypatch.setattr(goose_admission, "GOOSE_ARCHIVE_OBSERVED_BYTES", archive.stat().st_size)
|
|
monkeypatch.setattr(goose_admission, "_is_canonical_worker_root", lambda _: True)
|
|
|
|
manifest = admit_goose_validation(root, archive_path=archive, preview_points=2)
|
|
|
|
assert manifest["status"] == "frame-ready"
|
|
assert manifest["frame"]["frame_id"] == "2026_frame"
|
|
assert manifest["frame"]["point_count"] == 3
|
|
assert manifest["frame"]["ground_truth_ground_points"] == 2
|
|
state_path = root / "state/goose-3d-v2025-08-22.json"
|
|
assert read_dataset_admission_manifest(state_path)["status"] == "frame-ready"
|
|
digest = manifest["archive"]["sha256"]
|
|
preview_path = root / f"goose-3d/v2025-08-22/installs/{digest}/previews/2026_frame.json"
|
|
preview = read_dataset_native_scan_preview(preview_path)
|
|
assert preview["point_count"] == 2
|
|
assert len(preview["semantic_rgb_0_to_255"]) == 6
|
|
assert preview["safety"]["navigation_or_safety_accepted"] is False
|
|
|
|
|
|
def test_dataset_preview_api_is_read_only_and_integrity_checked(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
preview_path = tmp_path / "preview.json"
|
|
preview_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": "missioncore.dataset-native-scan-preview/v1",
|
|
"source_id": "goose-3d/v2025-08-22",
|
|
"frame_id": "frame-1",
|
|
"representation": "native-scan",
|
|
"sampling": "deterministic-even-index",
|
|
"source_point_count": 1,
|
|
"point_count": 1,
|
|
"points_xyz_m": [[1.0, 2.0, 3.0]],
|
|
"remission_0_to_255": [128],
|
|
"semantic_label_ids": [23],
|
|
"semantic_rgb_0_to_255": [255, 47, 128],
|
|
"ground_truth_ground": [1],
|
|
"classes": [
|
|
{
|
|
"label_id": 23,
|
|
"class_name": "asphalt",
|
|
"hex": "#ff2f80",
|
|
"challenge_category_id": 2,
|
|
"challenge_category_name": "artificial_ground",
|
|
}
|
|
],
|
|
"safety": {
|
|
"visualization_only": True,
|
|
"navigation_or_safety_accepted": False,
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
router = build_lidar_router(
|
|
dataset_admission_provider=lambda: None,
|
|
dataset_preview_provider=lambda: preview_path,
|
|
)
|
|
route = _endpoint(router, "/api/v1/lidar/dataset-gateway/preview")
|
|
|
|
response = route() # type: ignore[operator]
|
|
|
|
assert response["frame_id"] == "frame-1"
|
|
|
|
|
|
def test_goose_current_ground_is_scored_against_independent_labels(
|
|
tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
root = tmp_path / "datasets"
|
|
digest = "a" * 64
|
|
frame_id = "frame-1"
|
|
install = root / f"goose-3d/v2025-08-22/installs/{digest}"
|
|
frame_root = install / f"frames/{frame_id}"
|
|
frame_root.mkdir(parents=True)
|
|
np.asarray(
|
|
[
|
|
[-1.0, 0.0, 0.00, 0.1],
|
|
[0.0, 0.0, 0.02, 0.2],
|
|
[1.0, 0.0, 0.01, 0.3],
|
|
[0.0, 0.2, 1.50, 0.4],
|
|
],
|
|
dtype="<f4",
|
|
).tofile(frame_root / "points.bin")
|
|
np.asarray([23, 31, 23, 38], dtype="<u4").tofile(frame_root / "labels.label")
|
|
(install / "goose_label_mapping.csv").write_text(
|
|
"class_name,label_key,has_instance,hex\n"
|
|
"asphalt,23,0,#ff2f80\n"
|
|
"soil,31,0,#d16100\n"
|
|
"building,38,0,#013349\n",
|
|
encoding="utf-8",
|
|
)
|
|
state = {
|
|
"schema_version": "missioncore.dataset-admission/v1",
|
|
"source_id": "goose-3d/v2025-08-22",
|
|
"observed_at_utc": "2026-07-25T10:00:00Z",
|
|
"status": "frame-ready",
|
|
"storage": {
|
|
"policy": "worker-d-only",
|
|
"admitted": True,
|
|
"canonical_root": True,
|
|
"path_exposed": False,
|
|
},
|
|
"archive": {
|
|
"filename": "goose_3d_val.zip",
|
|
"source_url": "https://goose-dataset.de/storage/goose_3d_val.zip",
|
|
"bytes_transferred": 1024,
|
|
"total_bytes": 1024,
|
|
"size_bytes": 1024,
|
|
"sha256": digest,
|
|
"integrity": "zip-structure-and-content-digest",
|
|
"vendor_checksum_available": False,
|
|
},
|
|
"license": {"spdx": "CC-BY-SA-4.0", "artifact_present": True},
|
|
"frame": {
|
|
"frame_id": frame_id,
|
|
"representation": "native-scan",
|
|
"point_count": 4,
|
|
"semantic_class_count": 3,
|
|
"ground_truth_ground_points": 3,
|
|
"ground_truth_ground_fraction": 0.75,
|
|
"preview_point_count": 4,
|
|
"preview_sha256": "b" * 64,
|
|
"preview_available": True,
|
|
},
|
|
"next_action": "review-first-native-scan",
|
|
}
|
|
state_path = root / "state/goose-3d-v2025-08-22.json"
|
|
state_path.parent.mkdir(parents=True)
|
|
state_path.write_text(json.dumps(state), encoding="utf-8")
|
|
monkeypatch.setattr(goose_benchmark, "_is_worker_dataset_root", lambda _: True)
|
|
|
|
report = benchmark_goose_current_ground(root, preview_points=4)
|
|
|
|
assert report["ground_truth"]["evaluated_points"] == 4
|
|
assert 0 <= report["metrics"]["ground_iou"] <= 1
|
|
assert report["decision"]["promoted"] is False
|
|
output = install / "benchmarks" / f"current-ground-{report['identity_sha256']}" / "preview.json"
|
|
preview = json.loads(output.read_text(encoding="utf-8"))
|
|
assert preview["point_count"] == 4
|
|
assert len(preview["disagreement"]) == 4
|
|
|
|
class FakePatchwork:
|
|
identity = {
|
|
"provider_id": "patchworkpp/v1.4.1",
|
|
"source_url": "https://github.com/url-kaist/patchwork-plusplus",
|
|
"source_tag": "v1.4.1",
|
|
"source_commit": "3e6903a1d5537a4cc2ace897b0bbb98a92d6014c",
|
|
"binding_version": "test",
|
|
"binary_sha256": "c" * 64,
|
|
"platform": "linux",
|
|
"machine": "x86_64",
|
|
"ground_truth": False,
|
|
}
|
|
|
|
def segment(self, xyzi: np.ndarray) -> GroundSegmentation:
|
|
return GroundSegmentation(
|
|
ground_mask=np.asarray([True, True, False, False]),
|
|
assigned_mask=np.ones(xyzi.shape[0], dtype=np.bool_),
|
|
latency_ms=0.2,
|
|
)
|
|
|
|
ab_report = benchmark_goose_patchwork_ground(
|
|
root,
|
|
patchwork=FakePatchwork(),
|
|
preview_points=4,
|
|
)
|
|
|
|
assert ab_report["scope"]["patchworkpp_input_admitted"] is True
|
|
assert ab_report["scope"]["normalized_scan_produced"] is False
|
|
assert ab_report["decision"]["promoted"] is False
|
|
ab_output = (
|
|
install / "benchmarks" / f"ground-ab-{ab_report['identity_sha256']}" / "preview.json"
|
|
)
|
|
ab_preview = read_dataset_ground_preview(ab_output)
|
|
assert ab_preview["schema_version"] == "missioncore.dataset-ground-comparison-preview/v2"
|
|
assert ab_preview["input_profile"]["height"]["sensor_above_ground_m"] == 2.24
|
|
assert len(ab_preview["patchwork_disagreement"]) == 4
|
|
|
|
ab_preview["input_profile"]["height"]["sensor_above_ground_m"] = 2.25
|
|
ab_output.write_text(json.dumps(ab_preview), encoding="utf-8")
|
|
with pytest.raises(DatasetAdmissionError, match="input profile"):
|
|
read_dataset_ground_preview(ab_output)
|