feat(lidar): admit and benchmark GOOSE baseline
This commit is contained in:
DCCONSTRUCTIONS
@@ -2,16 +2,38 @@
|
||||
|
||||
from k1link.datasets.gateway import (
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||
DatasetAdmissionError,
|
||||
DatasetFrameError,
|
||||
DatasetPointFrame,
|
||||
configured_dataset_admission_manifest,
|
||||
configured_dataset_ground_preview,
|
||||
configured_dataset_preview,
|
||||
dataset_gateway_catalog,
|
||||
read_dataset_admission_manifest,
|
||||
read_dataset_ground_preview,
|
||||
read_dataset_native_scan_preview,
|
||||
read_semantic_kitti_frame,
|
||||
)
|
||||
from k1link.datasets.goose_benchmark import (
|
||||
GOOSE_GROUND_BENCHMARK_SCHEMA,
|
||||
GOOSE_GROUND_PREVIEW_SCHEMA,
|
||||
benchmark_goose_current_ground,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DATASET_GATEWAY_CATALOG_SCHEMA",
|
||||
"DatasetAdmissionError",
|
||||
"DatasetFrameError",
|
||||
"DatasetPointFrame",
|
||||
"GOOSE_GROUND_BENCHMARK_SCHEMA",
|
||||
"GOOSE_GROUND_PREVIEW_SCHEMA",
|
||||
"benchmark_goose_current_ground",
|
||||
"configured_dataset_admission_manifest",
|
||||
"configured_dataset_ground_preview",
|
||||
"configured_dataset_preview",
|
||||
"dataset_gateway_catalog",
|
||||
"read_dataset_admission_manifest",
|
||||
"read_dataset_ground_preview",
|
||||
"read_dataset_native_scan_preview",
|
||||
"read_semantic_kitti_frame",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Operator CLI for worker-local public dataset admission."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from k1link.datasets.goose_admission import GooseAdmissionError, admit_goose_validation
|
||||
from k1link.datasets.goose_benchmark import benchmark_goose_current_ground
|
||||
|
||||
app = typer.Typer(
|
||||
add_completion=False,
|
||||
help="Admit public perception datasets without moving source data off worker D.",
|
||||
)
|
||||
|
||||
|
||||
@app.command("admit-goose-validation")
|
||||
def admit_goose_validation_command(
|
||||
dataset_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||
],
|
||||
archive: Annotated[
|
||||
Path | None,
|
||||
typer.Option("--archive", exists=True, dir_okay=False, resolve_path=True),
|
||||
] = None,
|
||||
preview_points: Annotated[
|
||||
int,
|
||||
typer.Option("--preview-points", min=1, max=50_000),
|
||||
] = 50_000,
|
||||
) -> None:
|
||||
"""Verify the pinned GOOSE validation archive and import one native scan."""
|
||||
|
||||
try:
|
||||
manifest = admit_goose_validation(
|
||||
dataset_root,
|
||||
archive_path=archive,
|
||||
preview_points=preview_points,
|
||||
)
|
||||
except GooseAdmissionError as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
@app.command("benchmark-goose-current-ground")
|
||||
def benchmark_goose_current_ground_command(
|
||||
dataset_root: Annotated[
|
||||
Path,
|
||||
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
|
||||
],
|
||||
) -> None:
|
||||
"""Score the existing Mission Core ground baseline against GOOSE labels."""
|
||||
|
||||
try:
|
||||
report = benchmark_goose_current_ground(dataset_root)
|
||||
except GooseAdmissionError as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=2) from exc
|
||||
typer.echo(json.dumps(report, ensure_ascii=False, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
+389
-13
@@ -7,16 +7,25 @@ map construction are separate, provenance-bearing products.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PureWindowsPath
|
||||
from typing import Final, Literal
|
||||
from typing import Any, Final, Literal
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v1"
|
||||
DATASET_GATEWAY_CATALOG_SCHEMA: Final = "missioncore.dataset-gateway-catalog/v2"
|
||||
DATASET_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||
DATASET_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||
DATASET_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
||||
DATASET_ROOT_ENV: Final = "MISSIONCORE_DATASET_ROOT"
|
||||
DATASET_ADMISSION_MANIFEST_ENV: Final = "MISSIONCORE_DATASET_ADMISSION_MANIFEST"
|
||||
DATASET_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_PREVIEW"
|
||||
DATASET_GROUND_PREVIEW_ENV: Final = "MISSIONCORE_DATASET_GROUND_PREVIEW"
|
||||
MAX_STATE_BYTES: Final = 64 * 1024
|
||||
MAX_PREVIEW_BYTES: Final = 16 * 1024**2
|
||||
Representation = Literal["native-scan", "normalized-scan", "rolling-local-map"]
|
||||
|
||||
|
||||
@@ -24,6 +33,10 @@ class DatasetFrameError(ValueError):
|
||||
"""A dataset frame cannot satisfy its declared lossless source contract."""
|
||||
|
||||
|
||||
class DatasetAdmissionError(ValueError):
|
||||
"""A worker admission artifact violates the path-free dataset contract."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetPointFrame:
|
||||
"""One SemanticKITTI-compatible labeled LiDAR frame.
|
||||
@@ -91,6 +104,21 @@ def _configured_dataset_root() -> Path | None:
|
||||
return Path(raw).expanduser().absolute() if raw else None
|
||||
|
||||
|
||||
def configured_dataset_admission_manifest() -> Path | None:
|
||||
raw = os.environ.get(DATASET_ADMISSION_MANIFEST_ENV, "").strip()
|
||||
return Path(raw).expanduser().absolute() if raw else None
|
||||
|
||||
|
||||
def configured_dataset_preview() -> Path | None:
|
||||
raw = os.environ.get(DATASET_PREVIEW_ENV, "").strip()
|
||||
return Path(raw).expanduser().absolute() if raw else None
|
||||
|
||||
|
||||
def configured_dataset_ground_preview() -> Path | None:
|
||||
raw = os.environ.get(DATASET_GROUND_PREVIEW_ENV, "").strip()
|
||||
return Path(raw).expanduser().absolute() if raw else None
|
||||
|
||||
|
||||
def _is_worker_d_storage(root: Path | None) -> bool:
|
||||
if root is None:
|
||||
return False
|
||||
@@ -104,20 +132,61 @@ def _is_worker_d_storage(root: Path | None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, object]:
|
||||
def dataset_gateway_catalog(
|
||||
dataset_root: Path | None = None,
|
||||
admission_manifest_path: Path | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Return the path-free, read-only ingress plan and current admission state."""
|
||||
|
||||
root = dataset_root if dataset_root is not None else _configured_dataset_root()
|
||||
storage_admitted = _is_worker_d_storage(root)
|
||||
manifest_path = (
|
||||
admission_manifest_path
|
||||
if admission_manifest_path is not None
|
||||
else configured_dataset_admission_manifest()
|
||||
)
|
||||
admission: dict[str, Any] | None = None
|
||||
admission_error: str | None = None
|
||||
if manifest_path is not None:
|
||||
try:
|
||||
admission = read_dataset_admission_manifest(manifest_path)
|
||||
except DatasetAdmissionError:
|
||||
admission_error = "worker-manifest-invalid"
|
||||
locally_admitted = _is_worker_d_storage(root)
|
||||
worker_admitted = bool(admission and admission["storage"]["admitted"])
|
||||
storage_admitted = locally_admitted or worker_admitted
|
||||
source_status = (
|
||||
str(admission["status"])
|
||||
if admission is not None
|
||||
else "ready-for-download"
|
||||
if storage_admitted
|
||||
else "blocked-storage-policy"
|
||||
)
|
||||
next_action = (
|
||||
str(admission["next_action"])
|
||||
if admission is not None
|
||||
else "repair-worker-admission-manifest"
|
||||
if admission_error is not None
|
||||
else "download-goose-validation-to-d"
|
||||
if storage_admitted
|
||||
else "configure-dataset-root-on-worker-d"
|
||||
)
|
||||
return {
|
||||
"schema_version": DATASET_GATEWAY_CATALOG_SCHEMA,
|
||||
"access": "read-only",
|
||||
"storage": {
|
||||
"configured": root is not None,
|
||||
"configured": root is not None or manifest_path is not None,
|
||||
"required_windows_root": r"D:\NDC_MISSIONCORE\datasets",
|
||||
"required_wsl_root": "/mnt/d/NDC_MISSIONCORE/datasets",
|
||||
"admitted": storage_admitted,
|
||||
"status": "ready" if storage_admitted else "blocked-storage-policy",
|
||||
"attestation": (
|
||||
"worker-manifest"
|
||||
if worker_admitted
|
||||
else "local-worker-path"
|
||||
if locally_admitted
|
||||
else "none"
|
||||
),
|
||||
"manifest_valid": admission is not None,
|
||||
"path_exposed": False,
|
||||
},
|
||||
"sources": [
|
||||
@@ -149,12 +218,12 @@ def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, objec
|
||||
"test_archive_gb": 3.3,
|
||||
},
|
||||
"admission": {
|
||||
"status": (
|
||||
"ready-for-download" if storage_admitted else "blocked-storage-policy"
|
||||
),
|
||||
"status": source_status,
|
||||
"native_scan": "ready-after-download",
|
||||
"normalized_scan": "requires-explicit-frame-and-mounting-contract",
|
||||
"rolling_local_map": "requires-pose-timing-and-map-policy",
|
||||
"archive": admission["archive"] if admission is not None else None,
|
||||
"frame": admission["frame"] if admission is not None else None,
|
||||
},
|
||||
}
|
||||
],
|
||||
@@ -219,9 +288,316 @@ def dataset_gateway_catalog(dataset_root: Path | None = None) -> dict[str, objec
|
||||
"reason": "post-lio-map-product-cannot-be-reconstructed-as-a-native-scan",
|
||||
}
|
||||
],
|
||||
"next_action": (
|
||||
"download-goose-validation-to-d"
|
||||
if storage_admitted
|
||||
else "configure-dataset-root-on-worker-d"
|
||||
),
|
||||
"next_action": next_action,
|
||||
}
|
||||
|
||||
|
||||
def read_dataset_admission_manifest(path: Path) -> dict[str, Any]:
|
||||
document = _bounded_json_object(path, MAX_STATE_BYTES, "dataset admission manifest")
|
||||
if (
|
||||
document.get("schema_version") != DATASET_ADMISSION_SCHEMA
|
||||
or document.get("source_id") != "goose-3d/v2025-08-22"
|
||||
or set(document)
|
||||
!= {
|
||||
"schema_version",
|
||||
"source_id",
|
||||
"observed_at_utc",
|
||||
"status",
|
||||
"storage",
|
||||
"archive",
|
||||
"license",
|
||||
"frame",
|
||||
"next_action",
|
||||
}
|
||||
):
|
||||
raise DatasetAdmissionError("dataset admission manifest identity is incompatible")
|
||||
status = document["status"]
|
||||
if status not in {"downloading", "downloaded", "verifying", "verified", "frame-ready"}:
|
||||
raise DatasetAdmissionError("dataset admission status is incompatible")
|
||||
storage = _object(document["storage"], "storage")
|
||||
if (
|
||||
set(storage)
|
||||
!= {"policy", "admitted", "canonical_root", "path_exposed"}
|
||||
or storage.get("policy") != "worker-d-only"
|
||||
or storage.get("admitted") is not True
|
||||
or storage.get("canonical_root") is not True
|
||||
or storage.get("path_exposed") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset storage admission is incompatible")
|
||||
archive = _object(document["archive"], "archive")
|
||||
if (
|
||||
set(archive)
|
||||
!= {
|
||||
"filename",
|
||||
"source_url",
|
||||
"bytes_transferred",
|
||||
"total_bytes",
|
||||
"size_bytes",
|
||||
"sha256",
|
||||
"integrity",
|
||||
"vendor_checksum_available",
|
||||
}
|
||||
or archive.get("filename") != "goose_3d_val.zip"
|
||||
or archive.get("source_url")
|
||||
!= "https://goose-dataset.de/storage/goose_3d_val.zip"
|
||||
or archive.get("vendor_checksum_available") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset archive admission is incompatible")
|
||||
transferred = _nonnegative_integer(archive["bytes_transferred"], "bytes_transferred")
|
||||
total = _positive_integer(archive["total_bytes"], "total_bytes")
|
||||
if transferred > total:
|
||||
raise DatasetAdmissionError("dataset download progress is incompatible")
|
||||
size = archive["size_bytes"]
|
||||
digest = archive["sha256"]
|
||||
if size is not None:
|
||||
_positive_integer(size, "size_bytes")
|
||||
if digest is not None and (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset archive digest is incompatible")
|
||||
license_value = _object(document["license"], "license")
|
||||
if (
|
||||
set(license_value) != {"spdx", "artifact_present"}
|
||||
or license_value.get("spdx") != "CC-BY-SA-4.0"
|
||||
or not isinstance(license_value.get("artifact_present"), bool)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset license admission is incompatible")
|
||||
frame = document["frame"]
|
||||
if status == "frame-ready":
|
||||
_validate_frame_admission(_object(frame, "frame"))
|
||||
elif frame is not None:
|
||||
raise DatasetAdmissionError("dataset frame must be absent before frame-ready")
|
||||
for key in ("observed_at_utc", "next_action"):
|
||||
if not isinstance(document[key], str) or not document[key]:
|
||||
raise DatasetAdmissionError(f"{key} is incompatible")
|
||||
return document
|
||||
|
||||
|
||||
def read_dataset_native_scan_preview(path: Path) -> dict[str, Any]:
|
||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset preview")
|
||||
if (
|
||||
document.get("schema_version") != DATASET_PREVIEW_SCHEMA
|
||||
or document.get("source_id") != "goose-3d/v2025-08-22"
|
||||
or document.get("representation") != "native-scan"
|
||||
or document.get("sampling") != "deterministic-even-index"
|
||||
):
|
||||
raise DatasetAdmissionError("dataset preview identity is incompatible")
|
||||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||
source_point_count = _positive_integer(
|
||||
document.get("source_point_count"), "source_point_count"
|
||||
)
|
||||
if point_count > 50_000 or point_count > source_point_count:
|
||||
raise DatasetAdmissionError("dataset preview point count is incompatible")
|
||||
points = document.get("points_xyz_m")
|
||||
remission = document.get("remission_0_to_255")
|
||||
semantic_ids = document.get("semantic_label_ids")
|
||||
semantic_rgb = document.get("semantic_rgb_0_to_255")
|
||||
ground = document.get("ground_truth_ground")
|
||||
if (
|
||||
not isinstance(points, list)
|
||||
or len(points) != point_count
|
||||
or not isinstance(remission, list)
|
||||
or len(remission) != point_count
|
||||
or not isinstance(semantic_ids, list)
|
||||
or len(semantic_ids) != point_count
|
||||
or not isinstance(semantic_rgb, list)
|
||||
or len(semantic_rgb) != point_count * 3
|
||||
or not isinstance(ground, list)
|
||||
or len(ground) != point_count
|
||||
):
|
||||
raise DatasetAdmissionError("dataset preview arrays are not point-aligned")
|
||||
for point in points:
|
||||
if (
|
||||
not isinstance(point, list)
|
||||
or len(point) != 3
|
||||
or any(
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or not np.isfinite(value)
|
||||
for value in point
|
||||
)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset preview point is incompatible")
|
||||
for values, maximum, label in (
|
||||
(remission, 255, "remission"),
|
||||
(semantic_ids, 65_535, "semantic label"),
|
||||
(semantic_rgb, 255, "semantic color"),
|
||||
(ground, 1, "ground mask"),
|
||||
):
|
||||
if any(
|
||||
not isinstance(value, int) or isinstance(value, bool) or not 0 <= value <= maximum
|
||||
for value in values
|
||||
):
|
||||
raise DatasetAdmissionError(f"dataset preview {label} is incompatible")
|
||||
classes = document.get("classes")
|
||||
if not isinstance(classes, list) or len(classes) > 64:
|
||||
raise DatasetAdmissionError("dataset preview class catalog is incompatible")
|
||||
safety = _object(document.get("safety"), "safety")
|
||||
if safety != {
|
||||
"visualization_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}:
|
||||
raise DatasetAdmissionError("dataset preview safety boundary is incompatible")
|
||||
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
|
||||
raise DatasetAdmissionError("dataset preview frame id is incompatible")
|
||||
return document
|
||||
|
||||
|
||||
def read_dataset_ground_preview(path: Path) -> dict[str, Any]:
|
||||
document = _bounded_json_object(path, MAX_PREVIEW_BYTES, "dataset ground preview")
|
||||
if (
|
||||
document.get("schema_version") != DATASET_GROUND_PREVIEW_SCHEMA
|
||||
or document.get("source_id") != "goose-3d/v2025-08-22"
|
||||
or document.get("sampling") != "deterministic-even-index"
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground preview identity is incompatible")
|
||||
point_count = _positive_integer(document.get("point_count"), "point_count")
|
||||
if point_count > 50_000:
|
||||
raise DatasetAdmissionError("dataset ground preview point count is incompatible")
|
||||
for key in ("current_ground", "ground_truth_ground", "evaluated", "disagreement"):
|
||||
values = document.get(key)
|
||||
if (
|
||||
not isinstance(values, list)
|
||||
or len(values) != point_count
|
||||
or any(
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or value not in (0, 1)
|
||||
for value in values
|
||||
)
|
||||
):
|
||||
raise DatasetAdmissionError(f"dataset ground preview {key} is incompatible")
|
||||
metrics = _object(document.get("metrics"), "metrics")
|
||||
expected_metrics = {
|
||||
"true_positive",
|
||||
"false_positive",
|
||||
"false_negative",
|
||||
"true_negative",
|
||||
"precision",
|
||||
"recall",
|
||||
"f1",
|
||||
"ground_iou",
|
||||
"accuracy",
|
||||
"artificial_ground_recall",
|
||||
"natural_ground_recall",
|
||||
"obstacle_non_ground_recall",
|
||||
}
|
||||
if set(metrics) != expected_metrics:
|
||||
raise DatasetAdmissionError("dataset ground metrics are incompatible")
|
||||
for key, value in metrics.items():
|
||||
if key in {"true_positive", "false_positive", "false_negative", "true_negative"}:
|
||||
_nonnegative_integer(value, f"metrics.{key}")
|
||||
elif (
|
||||
not isinstance(value, (int, float))
|
||||
or isinstance(value, bool)
|
||||
or not np.isfinite(value)
|
||||
or not 0 <= value <= 1
|
||||
):
|
||||
raise DatasetAdmissionError(f"dataset ground metric {key} is incompatible")
|
||||
latency = document.get("latency_ms")
|
||||
if (
|
||||
not isinstance(latency, (int, float))
|
||||
or isinstance(latency, bool)
|
||||
or not np.isfinite(latency)
|
||||
or latency < 0
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground latency is incompatible")
|
||||
provider = _object(document.get("provider"), "provider")
|
||||
if (
|
||||
set(provider) != {"provider_id", "implementation_sha256", "ground_truth"}
|
||||
or provider.get("provider_id") != "missioncore-local-percentile-ground/v1"
|
||||
or provider.get("ground_truth") is not False
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground provider is incompatible")
|
||||
digest = provider.get("implementation_sha256")
|
||||
if (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground provider digest is incompatible")
|
||||
safety = _object(document.get("safety"), "safety")
|
||||
if safety != {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}:
|
||||
raise DatasetAdmissionError("dataset ground safety boundary is incompatible")
|
||||
if not isinstance(document.get("frame_id"), str) or not document["frame_id"]:
|
||||
raise DatasetAdmissionError("dataset ground frame id is incompatible")
|
||||
return document
|
||||
|
||||
|
||||
def _validate_frame_admission(frame: dict[str, Any]) -> None:
|
||||
if (
|
||||
set(frame)
|
||||
!= {
|
||||
"frame_id",
|
||||
"representation",
|
||||
"point_count",
|
||||
"semantic_class_count",
|
||||
"ground_truth_ground_points",
|
||||
"ground_truth_ground_fraction",
|
||||
"preview_point_count",
|
||||
"preview_sha256",
|
||||
"preview_available",
|
||||
}
|
||||
or frame.get("representation") != "native-scan"
|
||||
or frame.get("preview_available") is not True
|
||||
):
|
||||
raise DatasetAdmissionError("dataset frame admission is incompatible")
|
||||
point_count = _positive_integer(frame["point_count"], "frame.point_count")
|
||||
ground_count = _nonnegative_integer(
|
||||
frame["ground_truth_ground_points"], "frame.ground_truth_ground_points"
|
||||
)
|
||||
if ground_count > point_count:
|
||||
raise DatasetAdmissionError("dataset ground-truth count is incompatible")
|
||||
for key in ("semantic_class_count", "preview_point_count"):
|
||||
value = _positive_integer(frame[key], key)
|
||||
if value > point_count:
|
||||
raise DatasetAdmissionError(f"{key} is incompatible")
|
||||
fraction = frame["ground_truth_ground_fraction"]
|
||||
if (
|
||||
not isinstance(fraction, (int, float))
|
||||
or isinstance(fraction, bool)
|
||||
or not 0 <= fraction <= 1
|
||||
):
|
||||
raise DatasetAdmissionError("dataset ground-truth fraction is incompatible")
|
||||
digest = frame["preview_sha256"]
|
||||
if (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
raise DatasetAdmissionError("dataset preview digest is incompatible")
|
||||
if not isinstance(frame["frame_id"], str) or not frame["frame_id"]:
|
||||
raise DatasetAdmissionError("dataset frame id is incompatible")
|
||||
|
||||
|
||||
def _bounded_json_object(path: Path, maximum_bytes: int, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
if not path.is_file() or not 0 < path.stat().st_size <= maximum_bytes:
|
||||
raise DatasetAdmissionError(f"{label} size is incompatible")
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise DatasetAdmissionError(f"{label} cannot be decoded") from exc
|
||||
return _object(document, label)
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise DatasetAdmissionError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_integer(value: object, label: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise DatasetAdmissionError(f"{label} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def _nonnegative_integer(value: object, label: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
||||
raise DatasetAdmissionError(f"{label} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
@@ -0,0 +1,519 @@
|
||||
"""Fail-closed admission of one GOOSE 3D validation frame.
|
||||
|
||||
The large source archive and extracted native frame remain on the worker D
|
||||
drive. The only portable products are a path-free admission manifest and a
|
||||
bounded visualization preview; neither is an alternative copy of the dataset.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.datasets.gateway import DatasetFrameError, read_semantic_kitti_frame
|
||||
|
||||
GOOSE_SOURCE_ID: Final = "goose-3d/v2025-08-22"
|
||||
GOOSE_VALIDATION_URL: Final = "https://goose-dataset.de/storage/goose_3d_val.zip"
|
||||
GOOSE_LICENSE: Final = "CC-BY-SA-4.0"
|
||||
GOOSE_ADMISSION_SCHEMA: Final = "missioncore.dataset-admission/v1"
|
||||
GOOSE_PREVIEW_SCHEMA: Final = "missioncore.dataset-native-scan-preview/v1"
|
||||
GOOSE_ARCHIVE_FILENAME: Final = "goose_3d_val.zip"
|
||||
GOOSE_ARCHIVE_OBSERVED_BYTES: Final = 3_498_402_435
|
||||
MAX_ARCHIVE_ENTRIES: Final = 100_000
|
||||
MAX_UNCOMPRESSED_BYTES: Final = 128 * 1024**3
|
||||
MAX_METADATA_BYTES: Final = 4 * 1024**2
|
||||
MAX_PREVIEW_POINTS: Final = 50_000
|
||||
GOOSE_CHALLENGE_GROUPS: Final[dict[str, frozenset[str]]] = {
|
||||
"other": frozenset({"undefined", "ego_vehicle", "outlier"}),
|
||||
"artificial_structures": frozenset({"building", "wall", "bridge", "tunnel"}),
|
||||
"artificial_ground": frozenset(
|
||||
{
|
||||
"cobble",
|
||||
"bikeway",
|
||||
"pedestrian_crossing",
|
||||
"road_marking",
|
||||
"sidewalk",
|
||||
"curb",
|
||||
"asphalt",
|
||||
}
|
||||
),
|
||||
"natural_ground": frozenset(
|
||||
{"snow", "leaves", "gravel", "soil", "low_grass", "water"}
|
||||
),
|
||||
"obstacle": frozenset(
|
||||
{
|
||||
"traffic_cone",
|
||||
"obstacle",
|
||||
"street_light",
|
||||
"road_block",
|
||||
"traffic_light",
|
||||
"boom_barrier",
|
||||
"rail_track",
|
||||
"debris",
|
||||
"animal",
|
||||
"rock",
|
||||
"fence",
|
||||
"guard_rail",
|
||||
"pole",
|
||||
"traffic_sign",
|
||||
"misc_sign",
|
||||
"barrier_tape",
|
||||
"wire",
|
||||
"container",
|
||||
"barrel",
|
||||
"pipe",
|
||||
}
|
||||
),
|
||||
"vehicle": frozenset(
|
||||
{
|
||||
"car",
|
||||
"bicycle",
|
||||
"bus",
|
||||
"motorcycle",
|
||||
"truck",
|
||||
"on_rails",
|
||||
"caravan",
|
||||
"trailer",
|
||||
"kick_scooter",
|
||||
"heavy_machinery",
|
||||
"military_vehicle",
|
||||
}
|
||||
),
|
||||
"vegetation": frozenset(
|
||||
{
|
||||
"forest",
|
||||
"bush",
|
||||
"moss",
|
||||
"tree_crown",
|
||||
"tree_trunk",
|
||||
"crops",
|
||||
"high_grass",
|
||||
"scenery_vegetation",
|
||||
"hedge",
|
||||
"tree_root",
|
||||
}
|
||||
),
|
||||
"human": frozenset({"person", "rider"}),
|
||||
"sky": frozenset({"sky"}),
|
||||
}
|
||||
GOOSE_CHALLENGE_IDS: Final = {
|
||||
name: identifier
|
||||
for identifier, name in enumerate(
|
||||
(
|
||||
"other",
|
||||
"artificial_structures",
|
||||
"artificial_ground",
|
||||
"natural_ground",
|
||||
"obstacle",
|
||||
"vehicle",
|
||||
"vegetation",
|
||||
"human",
|
||||
"sky",
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class GooseAdmissionError(RuntimeError):
|
||||
"""The source archive cannot satisfy the pinned GOOSE admission contract."""
|
||||
|
||||
|
||||
def admit_goose_validation(
|
||||
dataset_root: Path,
|
||||
*,
|
||||
archive_path: Path | None = None,
|
||||
preview_points: int = MAX_PREVIEW_POINTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Verify a GOOSE validation archive and admit one deterministic frame."""
|
||||
|
||||
root = dataset_root.expanduser().absolute()
|
||||
if not _is_canonical_worker_root(root):
|
||||
raise GooseAdmissionError("GOOSE admission requires the canonical worker D dataset root")
|
||||
archive = (
|
||||
archive_path.expanduser().absolute()
|
||||
if archive_path is not None
|
||||
else root / "goose-3d/v2025-08-22/archives" / GOOSE_ARCHIVE_FILENAME
|
||||
)
|
||||
if not archive.is_file():
|
||||
raise GooseAdmissionError("GOOSE validation archive is unavailable")
|
||||
size_bytes = archive.stat().st_size
|
||||
if size_bytes != GOOSE_ARCHIVE_OBSERVED_BYTES:
|
||||
raise GooseAdmissionError(
|
||||
"GOOSE validation archive size differs from the pinned observed release"
|
||||
)
|
||||
if not 1 <= preview_points <= MAX_PREVIEW_POINTS:
|
||||
raise GooseAdmissionError("preview point limit is outside the admitted range")
|
||||
|
||||
archive_sha256 = _sha256_file(archive)
|
||||
install_root = root / "goose-3d/v2025-08-22/installs" / archive_sha256
|
||||
state_root = root / "state"
|
||||
state_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as source:
|
||||
members = _admitted_members(source)
|
||||
point_member, label_member = _first_frame_pair(members)
|
||||
mapping_member = _unique_metadata_member(members, "goose_label_mapping.csv")
|
||||
license_member = _unique_metadata_member(members, "LICENSE")
|
||||
mapping_bytes = _bounded_member_bytes(source, mapping_member)
|
||||
license_bytes = _bounded_member_bytes(source, license_member)
|
||||
labels = parse_goose_label_mapping(mapping_bytes)
|
||||
license_text = license_bytes.decode("utf-8", errors="replace")
|
||||
if "Attribution-ShareAlike 4.0 International" not in license_text:
|
||||
raise GooseAdmissionError("archive LICENSE does not declare CC BY-SA 4.0")
|
||||
|
||||
frame_id = _frame_id(point_member.filename)
|
||||
frame_root = install_root / "frames" / frame_id
|
||||
frame_root.mkdir(parents=True, exist_ok=True)
|
||||
point_path = frame_root / "points.bin"
|
||||
label_path = frame_root / "labels.label"
|
||||
mapping_path = install_root / "goose_label_mapping.csv"
|
||||
license_path = install_root / "LICENSE"
|
||||
_extract_once(source, point_member, point_path)
|
||||
_extract_once(source, label_member, label_path)
|
||||
_write_once(mapping_path, mapping_bytes)
|
||||
_write_once(license_path, license_bytes)
|
||||
except (OSError, zipfile.BadZipFile) as exc:
|
||||
raise GooseAdmissionError("GOOSE archive could not be verified") from exc
|
||||
|
||||
try:
|
||||
frame = read_semantic_kitti_frame(point_path, label_path)
|
||||
except DatasetFrameError as exc:
|
||||
raise GooseAdmissionError("first GOOSE frame violates XYZI/label alignment") from exc
|
||||
unknown_labels = sorted(
|
||||
int(value) for value in np.unique(frame.semantic_labels) if int(value) not in labels
|
||||
)
|
||||
if unknown_labels:
|
||||
raise GooseAdmissionError("frame contains semantic ids absent from the label mapping")
|
||||
|
||||
preview = _native_scan_preview(
|
||||
frame_id,
|
||||
frame.points_xyz_m,
|
||||
frame.remission,
|
||||
frame.semantic_labels,
|
||||
labels,
|
||||
maximum_points=preview_points,
|
||||
)
|
||||
preview_path = install_root / "previews" / f"{frame_id}.json"
|
||||
_atomic_json(preview_path, preview)
|
||||
preview_sha256 = _sha256_file(preview_path)
|
||||
present_classes = Counter(int(value) for value in frame.semantic_labels)
|
||||
ground_points = sum(
|
||||
count
|
||||
for label_id, count in present_classes.items()
|
||||
if labels[label_id]["challenge_category_id"] in (2, 3)
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": GOOSE_ADMISSION_SCHEMA,
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
"status": "frame-ready",
|
||||
"storage": {
|
||||
"policy": "worker-d-only",
|
||||
"admitted": True,
|
||||
"canonical_root": True,
|
||||
"path_exposed": False,
|
||||
},
|
||||
"archive": {
|
||||
"filename": GOOSE_ARCHIVE_FILENAME,
|
||||
"source_url": GOOSE_VALIDATION_URL,
|
||||
"bytes_transferred": size_bytes,
|
||||
"total_bytes": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"sha256": archive_sha256,
|
||||
"integrity": "zip-structure-and-content-digest",
|
||||
"vendor_checksum_available": False,
|
||||
},
|
||||
"license": {
|
||||
"spdx": GOOSE_LICENSE,
|
||||
"artifact_present": True,
|
||||
},
|
||||
"frame": {
|
||||
"frame_id": frame_id,
|
||||
"representation": "native-scan",
|
||||
"point_count": frame.point_count,
|
||||
"semantic_class_count": len(present_classes),
|
||||
"ground_truth_ground_points": ground_points,
|
||||
"ground_truth_ground_fraction": ground_points / frame.point_count,
|
||||
"preview_point_count": int(preview["point_count"]),
|
||||
"preview_sha256": preview_sha256,
|
||||
"preview_available": True,
|
||||
},
|
||||
"next_action": "review-first-native-scan",
|
||||
}
|
||||
_atomic_json(state_root / "goose-3d-v2025-08-22.json", manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _is_canonical_worker_root(root: Path) -> bool:
|
||||
normalized = str(root).replace("\\", "/").rstrip("/").lower()
|
||||
return normalized == "/mnt/d/ndc_missioncore/datasets"
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
raise GooseAdmissionError("dataset artifact cannot be hashed") from exc
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _admitted_members(source: zipfile.ZipFile) -> tuple[zipfile.ZipInfo, ...]:
|
||||
members = tuple(source.infolist())
|
||||
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
|
||||
raise GooseAdmissionError("archive entry count is outside the admitted range")
|
||||
total_bytes = 0
|
||||
for member in members:
|
||||
path = PurePosixPath(member.filename.replace("\\", "/"))
|
||||
if path.is_absolute() or ".." in path.parts or not path.parts:
|
||||
raise GooseAdmissionError("archive contains an unsafe member path")
|
||||
mode = member.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
raise GooseAdmissionError("archive contains a symbolic link")
|
||||
total_bytes += member.file_size
|
||||
if total_bytes > MAX_UNCOMPRESSED_BYTES:
|
||||
raise GooseAdmissionError("archive expands beyond the admitted size")
|
||||
return members
|
||||
|
||||
|
||||
def _first_frame_pair(
|
||||
members: tuple[zipfile.ZipInfo, ...],
|
||||
) -> tuple[zipfile.ZipInfo, zipfile.ZipInfo]:
|
||||
points = {
|
||||
_frame_id(member.filename): member
|
||||
for member in members
|
||||
if not member.is_dir()
|
||||
and any(
|
||||
prefix in "/" + member.filename.replace("\\", "/")
|
||||
for prefix in ("/lidar/val/", "/velodyne/val/")
|
||||
)
|
||||
and member.filename.endswith("_vls128.bin")
|
||||
}
|
||||
labels = {
|
||||
_frame_id(member.filename): member
|
||||
for member in members
|
||||
if not member.is_dir()
|
||||
and "/labels/val/" in "/" + member.filename.replace("\\", "/")
|
||||
and member.filename.endswith("_goose.label")
|
||||
}
|
||||
shared = sorted(points.keys() & labels.keys())
|
||||
if not shared:
|
||||
raise GooseAdmissionError("archive has no aligned GOOSE validation frame")
|
||||
frame_id = shared[0]
|
||||
return points[frame_id], labels[frame_id]
|
||||
|
||||
|
||||
def _frame_id(filename: str) -> str:
|
||||
name = PurePosixPath(filename.replace("\\", "/")).name
|
||||
for suffix in ("_vls128.bin", "_goose.label"):
|
||||
if name.endswith(suffix):
|
||||
identifier = name[: -len(suffix)]
|
||||
if identifier:
|
||||
return identifier
|
||||
raise GooseAdmissionError("GOOSE frame filename is incompatible")
|
||||
|
||||
|
||||
def _unique_metadata_member(
|
||||
members: tuple[zipfile.ZipInfo, ...],
|
||||
filename: str,
|
||||
) -> zipfile.ZipInfo:
|
||||
matches = [member for member in members if PurePosixPath(member.filename).name == filename]
|
||||
if len(matches) != 1 or matches[0].is_dir():
|
||||
raise GooseAdmissionError(f"archive must contain exactly one {filename}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _bounded_member_bytes(source: zipfile.ZipFile, member: zipfile.ZipInfo) -> bytes:
|
||||
if member.file_size <= 0 or member.file_size > MAX_METADATA_BYTES:
|
||||
raise GooseAdmissionError("archive metadata size is outside the admitted range")
|
||||
value = source.read(member)
|
||||
if len(value) != member.file_size:
|
||||
raise GooseAdmissionError("archive metadata was truncated")
|
||||
return value
|
||||
|
||||
|
||||
def _extract_once(source: zipfile.ZipFile, member: zipfile.ZipInfo, target: Path) -> None:
|
||||
if target.exists():
|
||||
if target.is_file() and target.stat().st_size == member.file_size:
|
||||
return
|
||||
raise GooseAdmissionError("immutable extracted artifact already exists with another size")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
try:
|
||||
with source.open(member) as member_source:
|
||||
shutil.copyfileobj(member_source, temporary, length=1024**2)
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
except Exception:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
if temporary_path.stat().st_size != member.file_size:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise GooseAdmissionError("extracted artifact size does not match the archive")
|
||||
os.replace(temporary_path, target)
|
||||
|
||||
|
||||
def _write_once(target: Path, value: bytes) -> None:
|
||||
if target.exists():
|
||||
if target.is_file() and target.read_bytes() == value:
|
||||
return
|
||||
raise GooseAdmissionError("immutable metadata artifact already exists with other content")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary.write(value)
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
os.replace(temporary_path, target)
|
||||
|
||||
|
||||
def read_goose_label_mapping(path: Path) -> dict[int, dict[str, Any]]:
|
||||
try:
|
||||
value = path.read_bytes()
|
||||
except OSError as exc:
|
||||
raise GooseAdmissionError("GOOSE label mapping is unavailable") from exc
|
||||
if not 0 < len(value) <= MAX_METADATA_BYTES:
|
||||
raise GooseAdmissionError("GOOSE label mapping size is outside the admitted range")
|
||||
return parse_goose_label_mapping(value)
|
||||
|
||||
|
||||
def parse_goose_label_mapping(value: bytes) -> dict[int, dict[str, Any]]:
|
||||
try:
|
||||
text = value.decode("utf-8-sig")
|
||||
rows = tuple(csv.DictReader(text.splitlines()))
|
||||
except (UnicodeDecodeError, csv.Error) as exc:
|
||||
raise GooseAdmissionError("GOOSE label mapping cannot be decoded") from exc
|
||||
required = {
|
||||
"class_name",
|
||||
"label_key",
|
||||
"hex",
|
||||
}
|
||||
if not rows or not required.issubset(rows[0]):
|
||||
raise GooseAdmissionError("GOOSE label mapping columns are incompatible")
|
||||
labels: dict[int, dict[str, Any]] = {}
|
||||
try:
|
||||
for row in rows:
|
||||
label_id = int(row["label_key"])
|
||||
color = row["hex"].strip().lower()
|
||||
if label_id in labels or not 0 <= label_id <= 65_535:
|
||||
raise GooseAdmissionError("GOOSE label ids are invalid")
|
||||
if len(color) != 7 or color[0] != "#":
|
||||
raise GooseAdmissionError("GOOSE label color is invalid")
|
||||
int(color[1:], 16)
|
||||
labels[label_id] = {
|
||||
"class_name": row["class_name"].strip(),
|
||||
"hex": color,
|
||||
**_challenge_category(row["class_name"].strip()),
|
||||
}
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise GooseAdmissionError("GOOSE label mapping rows are invalid") from exc
|
||||
return labels
|
||||
|
||||
|
||||
def _challenge_category(class_name: str) -> dict[str, Any]:
|
||||
matches = [
|
||||
name for name, members in GOOSE_CHALLENGE_GROUPS.items() if class_name in members
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise GooseAdmissionError("GOOSE class is absent from the pinned challenge mapping")
|
||||
name = matches[0]
|
||||
return {
|
||||
"challenge_category_id": GOOSE_CHALLENGE_IDS[name],
|
||||
"challenge_category_name": name,
|
||||
}
|
||||
|
||||
|
||||
def _native_scan_preview(
|
||||
frame_id: str,
|
||||
points_xyz_m: np.ndarray[Any, Any],
|
||||
remission: np.ndarray[Any, Any],
|
||||
semantic_labels: np.ndarray[Any, Any],
|
||||
labels: dict[int, dict[str, Any]],
|
||||
*,
|
||||
maximum_points: int,
|
||||
) -> dict[str, Any]:
|
||||
point_count = int(points_xyz_m.shape[0])
|
||||
sample_count = min(point_count, maximum_points)
|
||||
indices = np.linspace(0, point_count - 1, sample_count, dtype=np.int64)
|
||||
sampled_points = points_xyz_m[indices]
|
||||
sampled_remission = remission[indices]
|
||||
sampled_labels = semantic_labels[indices]
|
||||
remission_min = float(np.min(sampled_remission))
|
||||
remission_max = float(np.max(sampled_remission))
|
||||
remission_span = remission_max - remission_min
|
||||
if not math.isfinite(remission_span):
|
||||
raise GooseAdmissionError("GOOSE remission range is non-finite")
|
||||
if remission_span > 0:
|
||||
scaled_remission = np.rint(
|
||||
(sampled_remission - remission_min) * (255.0 / remission_span)
|
||||
).astype(np.uint8)
|
||||
else:
|
||||
scaled_remission = np.zeros(sample_count, dtype=np.uint8)
|
||||
semantic_rgb: list[int] = []
|
||||
ground_mask: list[int] = []
|
||||
for value in sampled_labels:
|
||||
label = labels[int(value)]
|
||||
color = label["hex"]
|
||||
semantic_rgb.extend(
|
||||
(int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16))
|
||||
)
|
||||
ground_mask.append(1 if label["challenge_category_id"] in (2, 3) else 0)
|
||||
present_ids = sorted({int(value) for value in sampled_labels})
|
||||
return {
|
||||
"schema_version": GOOSE_PREVIEW_SCHEMA,
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"frame_id": frame_id,
|
||||
"representation": "native-scan",
|
||||
"sampling": "deterministic-even-index",
|
||||
"source_point_count": point_count,
|
||||
"point_count": sample_count,
|
||||
"points_xyz_m": sampled_points.tolist(),
|
||||
"remission_0_to_255": scaled_remission.tolist(),
|
||||
"semantic_label_ids": sampled_labels.astype(np.uint16).tolist(),
|
||||
"semantic_rgb_0_to_255": semantic_rgb,
|
||||
"ground_truth_ground": ground_mask,
|
||||
"classes": [
|
||||
{
|
||||
"label_id": label_id,
|
||||
**labels[label_id],
|
||||
}
|
||||
for label_id in present_ids
|
||||
],
|
||||
"safety": {
|
||||
"visualization_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
encoded = (
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode(
|
||||
"utf-8"
|
||||
)
|
||||
+ b"\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary.write(encoded)
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Independent-label ground benchmark for an admitted GOOSE native scan."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.datasets.gateway import read_dataset_admission_manifest, read_semantic_kitti_frame
|
||||
from k1link.datasets.goose_admission import (
|
||||
GOOSE_SOURCE_ID,
|
||||
GooseAdmissionError,
|
||||
read_goose_label_mapping,
|
||||
)
|
||||
from k1link.ground_segmentation import (
|
||||
DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
GroundBenchmarkProfile,
|
||||
LocalPercentileGroundSegmenter,
|
||||
)
|
||||
|
||||
GOOSE_GROUND_BENCHMARK_SCHEMA: Final = "missioncore.goose-ground-benchmark/v1"
|
||||
GOOSE_GROUND_PREVIEW_SCHEMA: Final = "missioncore.dataset-ground-comparison-preview/v1"
|
||||
MAX_PREVIEW_POINTS: Final = 50_000
|
||||
|
||||
|
||||
def benchmark_goose_current_ground(
|
||||
dataset_root: Path,
|
||||
*,
|
||||
profile: GroundBenchmarkProfile = DEFAULT_GROUND_BENCHMARK_PROFILE,
|
||||
preview_points: int = MAX_PREVIEW_POINTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Score the existing Mission Core ground baseline on one admitted frame."""
|
||||
|
||||
root = dataset_root.expanduser().absolute()
|
||||
if not _is_worker_dataset_root(root):
|
||||
raise GooseAdmissionError("GOOSE benchmark requires the canonical worker D root")
|
||||
manifest = read_dataset_admission_manifest(root / "state/goose-3d-v2025-08-22.json")
|
||||
if manifest["status"] != "frame-ready":
|
||||
raise GooseAdmissionError("GOOSE frame is not admitted for benchmarking")
|
||||
archive_sha256 = str(manifest["archive"]["sha256"])
|
||||
frame_id = str(manifest["frame"]["frame_id"])
|
||||
install = root / "goose-3d/v2025-08-22/installs" / archive_sha256
|
||||
frame_root = install / "frames" / frame_id
|
||||
frame = read_semantic_kitti_frame(
|
||||
frame_root / "points.bin",
|
||||
frame_root / "labels.label",
|
||||
)
|
||||
labels = read_goose_label_mapping(install / "goose_label_mapping.csv")
|
||||
categories = np.asarray(
|
||||
[labels[int(value)]["challenge_category_id"] for value in frame.semantic_labels],
|
||||
dtype=np.uint8,
|
||||
)
|
||||
evaluated = categories != 0
|
||||
ground_truth = (categories == 2) | (categories == 3)
|
||||
xyzi = np.column_stack((frame.points_xyz_m, frame.remission)).astype(
|
||||
np.float32,
|
||||
copy=False,
|
||||
)
|
||||
segmenter = LocalPercentileGroundSegmenter(profile)
|
||||
started = time.perf_counter_ns()
|
||||
prediction = segmenter.segment(xyzi)
|
||||
wall_latency_ms = (time.perf_counter_ns() - started) / 1_000_000
|
||||
metrics = _binary_metrics(
|
||||
prediction.ground_mask,
|
||||
ground_truth,
|
||||
evaluated,
|
||||
)
|
||||
metrics.update(
|
||||
{
|
||||
"artificial_ground_recall": _recall(
|
||||
prediction.ground_mask,
|
||||
categories == 2,
|
||||
),
|
||||
"natural_ground_recall": _recall(
|
||||
prediction.ground_mask,
|
||||
categories == 3,
|
||||
),
|
||||
"obstacle_non_ground_recall": _recall(
|
||||
~prediction.ground_mask,
|
||||
categories == 4,
|
||||
),
|
||||
}
|
||||
)
|
||||
profile_document = profile.to_dict()
|
||||
identity_document = {
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"archive_sha256": archive_sha256,
|
||||
"frame_id": frame_id,
|
||||
"source_point_count": frame.point_count,
|
||||
"profile": profile_document,
|
||||
"provider": dict(segmenter.identity),
|
||||
}
|
||||
identity_sha256 = _canonical_sha256(identity_document)
|
||||
output_root = install / "benchmarks" / f"current-ground-{identity_sha256}"
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
report_path = output_root / "report.json"
|
||||
if report_path.is_file():
|
||||
try:
|
||||
existing = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise GooseAdmissionError("existing GOOSE benchmark report is invalid") from exc
|
||||
if (
|
||||
not isinstance(existing, dict)
|
||||
or existing.get("schema_version") != GOOSE_GROUND_BENCHMARK_SCHEMA
|
||||
or existing.get("identity_sha256") != identity_sha256
|
||||
or not (output_root / "prediction.npz").is_file()
|
||||
or not (output_root / "preview.json").is_file()
|
||||
):
|
||||
raise GooseAdmissionError("existing GOOSE benchmark is incomplete")
|
||||
return existing
|
||||
prediction_path = output_root / "prediction.npz"
|
||||
_atomic_npz(
|
||||
prediction_path,
|
||||
current_ground=prediction.ground_mask.astype(np.uint8),
|
||||
ground_truth_ground=ground_truth.astype(np.uint8),
|
||||
evaluated=evaluated.astype(np.uint8),
|
||||
)
|
||||
prediction_sha256 = _sha256_file(prediction_path)
|
||||
|
||||
sample_count = min(frame.point_count, preview_points)
|
||||
indices = np.linspace(0, frame.point_count - 1, sample_count, dtype=np.int64)
|
||||
preview = {
|
||||
"schema_version": GOOSE_GROUND_PREVIEW_SCHEMA,
|
||||
"source_id": GOOSE_SOURCE_ID,
|
||||
"frame_id": frame_id,
|
||||
"sampling": "deterministic-even-index",
|
||||
"point_count": sample_count,
|
||||
"current_ground": prediction.ground_mask[indices].astype(np.uint8).tolist(),
|
||||
"ground_truth_ground": ground_truth[indices].astype(np.uint8).tolist(),
|
||||
"evaluated": evaluated[indices].astype(np.uint8).tolist(),
|
||||
"disagreement": (
|
||||
evaluated[indices]
|
||||
& (prediction.ground_mask[indices] != ground_truth[indices])
|
||||
)
|
||||
.astype(np.uint8)
|
||||
.tolist(),
|
||||
"metrics": metrics,
|
||||
"latency_ms": prediction.latency_ms,
|
||||
"provider": dict(segmenter.identity),
|
||||
"safety": {
|
||||
"qualification_only": True,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
preview_path = output_root / "preview.json"
|
||||
_atomic_json(preview_path, preview)
|
||||
report = {
|
||||
"schema_version": GOOSE_GROUND_BENCHMARK_SCHEMA,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity_document,
|
||||
"ground_truth": {
|
||||
"source": "GOOSE point-wise semantic labels",
|
||||
"ground_categories": ["artificial_ground", "natural_ground"],
|
||||
"void_excluded": True,
|
||||
"evaluated_points": int(np.count_nonzero(evaluated)),
|
||||
},
|
||||
"metrics": metrics,
|
||||
"latency": {
|
||||
"provider_ms": prediction.latency_ms,
|
||||
"wall_ms": wall_latency_ms,
|
||||
},
|
||||
"artifacts": {
|
||||
"prediction_sha256": prediction_sha256,
|
||||
"preview_sha256": _sha256_file(preview_path),
|
||||
},
|
||||
"decision": {
|
||||
"status": "diagnostic-baseline",
|
||||
"promoted": False,
|
||||
"reason": "one public frame does not qualify a production ground provider",
|
||||
},
|
||||
}
|
||||
_atomic_json(report_path, report)
|
||||
return report
|
||||
|
||||
|
||||
def _is_worker_dataset_root(root: Path) -> bool:
|
||||
normalized = str(root).replace("\\", "/").rstrip("/").lower()
|
||||
return normalized == "/mnt/d/ndc_missioncore/datasets"
|
||||
|
||||
|
||||
def _binary_metrics(
|
||||
prediction: np.ndarray[Any, Any],
|
||||
ground_truth: np.ndarray[Any, Any],
|
||||
evaluated: np.ndarray[Any, Any],
|
||||
) -> dict[str, float | int]:
|
||||
predicted = prediction & evaluated
|
||||
truth = ground_truth & evaluated
|
||||
true_positive = int(np.count_nonzero(predicted & truth))
|
||||
false_positive = int(np.count_nonzero(predicted & ~truth & evaluated))
|
||||
false_negative = int(np.count_nonzero(~predicted & truth))
|
||||
true_negative = int(np.count_nonzero(~predicted & ~truth & evaluated))
|
||||
precision = _ratio(true_positive, true_positive + false_positive)
|
||||
recall = _ratio(true_positive, true_positive + false_negative)
|
||||
return {
|
||||
"true_positive": true_positive,
|
||||
"false_positive": false_positive,
|
||||
"false_negative": false_negative,
|
||||
"true_negative": true_negative,
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"f1": _ratio(2 * precision * recall, precision + recall),
|
||||
"ground_iou": _ratio(
|
||||
true_positive,
|
||||
true_positive + false_positive + false_negative,
|
||||
),
|
||||
"accuracy": _ratio(
|
||||
true_positive + true_negative,
|
||||
true_positive + true_negative + false_positive + false_negative,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _recall(prediction: np.ndarray[Any, Any], truth: np.ndarray[Any, Any]) -> float:
|
||||
return _ratio(
|
||||
int(np.count_nonzero(prediction & truth)),
|
||||
int(np.count_nonzero(truth)),
|
||||
)
|
||||
|
||||
|
||||
def _ratio(numerator: float | int, denominator: float | int) -> float:
|
||||
return float(numerator / denominator) if denominator else 0.0
|
||||
|
||||
|
||||
def _canonical_sha256(value: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(1024**2), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _atomic_npz(path: Path, **arrays: np.ndarray[Any, Any]) -> None:
|
||||
if path.exists():
|
||||
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
|
||||
with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".npz", delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
try:
|
||||
np.savez_compressed(temporary_path, **arrays) # type: ignore[arg-type]
|
||||
with temporary_path.open("rb") as source:
|
||||
os.fsync(source.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
except Exception:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
||||
if path.exists():
|
||||
raise GooseAdmissionError("immutable GOOSE benchmark artifact already exists")
|
||||
encoded = (
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as temporary:
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary.write(encoded)
|
||||
temporary.flush()
|
||||
os.fsync(temporary.fileno())
|
||||
os.replace(temporary_path, path)
|
||||
Reference in New Issue
Block a user