diff --git a/experiments/perception/l3_dataset_inventory_2026-07-30.json b/experiments/perception/l3_dataset_inventory_2026-07-30.json new file mode 100644 index 0000000..b002d02 --- /dev/null +++ b/experiments/perception/l3_dataset_inventory_2026-07-30.json @@ -0,0 +1,67 @@ +{ + "datasets": [ + { + "annotations": [ + "point-semantic-labels", + "point-instance-labels" + ], + "dataset_id": "goose-3d/v2025-08-22", + "independent_ground_truth": true, + "installed": true, + "license": "CC-BY-SA-4.0", + "point_fields": [ + "x", + "y", + "z", + "intensity" + ], + "release_identity_sha256": "0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f", + "splits": [ + "validation" + ] + }, + { + "annotations": [ + "point-semantic-labels", + "point-instance-labels" + ], + "dataset_id": "rellis-3d/v1.1", + "independent_ground_truth": true, + "installed": true, + "license": "CC-BY-NC-SA-3.0", + "point_fields": [ + "x", + "y", + "z", + "intensity" + ], + "release_identity_sha256": "a3f3f161a5a7edccdf66cea75ecf9004b8f6a895282faabe661c075c15a82774", + "splits": [ + "train", + "validation", + "test" + ] + }, + { + "annotations": [ + "oriented-3d-boxes" + ], + "dataset_id": "kitti-3d-object/v2017", + "independent_ground_truth": true, + "installed": true, + "license": "CC-BY-NC-SA-3.0", + "point_fields": [ + "x", + "y", + "z", + "intensity" + ], + "release_identity_sha256": "2c9615bedca56b492b204b614d4419db6431626e2a6867e207a95999beefcf47", + "splits": [ + "validation" + ] + } + ], + "observed_at_utc": "2026-07-31T06:45:42.249612Z", + "schema_version": "missioncore.l3-lidar-dataset-inventory/v1" +} diff --git a/experiments/perception/l3_pointpillars_benchmark_profile.json b/experiments/perception/l3_pointpillars_benchmark_profile.json new file mode 100644 index 0000000..3eef579 --- /dev/null +++ b/experiments/perception/l3_pointpillars_benchmark_profile.json @@ -0,0 +1,139 @@ +{ + "authority": { + "commands_enabled": false, + "navigation_or_safety_accepted": false, + "shadow_only": true + }, + "detector": { + "batch_size": 1, + "candidate_frozen": true, + "candidate_model_version": "deployable_v1.1", + "candidate_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da", + "candidate_source_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1", + "model_classes": [ + "Vehicle", + "Pedestrian", + "Cyclist" + ], + "onnx_contract_sha256": "2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be", + "family": "nvidia-tao-pointpillars", + "input_coordinate_frame": "sensor/lidar", + "input_fields": [ + "x", + "y", + "z", + "intensity" + ], + "input_representation": "native-sensor-scan", + "maximum_points": 204800, + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432 + ], + "postprocessing": { + "class_agnostic_nms": true, + "embedded_contract_source": "onnx-node-attributes", + "embedded_score_threshold": 0.1, + "nms_iou_threshold": 0.01, + "output_row_fields": [ + "x", + "y", + "z", + "length", + "width", + "height", + "yaw", + "class_id", + "score" + ], + "pre_nms_top_n": 4096, + "reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8", + "reference_repository": "https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes" + }, + "required_source_format": "onnx", + "training_domain": "proprietary-solid-state-lidar", + "training_ground_truth_publicly_reproducible": false, + "triton_model_name": "pointpillars", + "upstream_model_id": "nvidia/tao/pointpillarnet" + }, + "k1_transfer_stability": { + "accuracy_claim_allowed": false, + "metrics": [ + "input-admission-rate", + "output-schema-valid-rate", + "deterministic-replay-rate", + "end-to-end-latency-ms", + "queue-wait-ms", + "drop-rate" + ], + "requires_completed_public_cross_domain_probe": true, + "retuning_allowed": false + }, + "public_cross_domain_probe": { + "benchmark_classes": [ + "Car", + "Pedestrian", + "Cyclist" + ], + "independent_ground_truth_required": true, + "metrics": [ + "bev-map", + "3d-map", + "center-error-m", + "range-error-m", + "yaw-error-rad", + "distance-bucket-recall", + "false-occupied-rate", + "end-to-end-latency-ms" + ], + "metric_contract": { + "ap_interpolation": "40-point", + "difficulty_filtering": false, + "distance_buckets_m": [ + [ + 0, + 20 + ], + [ + 20, + 40 + ], + [ + 40, + 70 + ] + ], + "iou_thresholds": { + "Car": 0.7, + "Cyclist": 0.5, + "Pedestrian": 0.5 + }, + "evaluation_kind": "public-cross-domain-transfer-probe", + "official_kitti_server_metric": false, + "predictions_outside_shared_range_ignored": true + }, + "model_to_benchmark_class_mapping": { + "Cyclist": "Cyclist", + "Pedestrian": "Pedestrian", + "Vehicle": "Car" + }, + "native_accuracy_claim_allowed": false, + "required_ground_truth": "oriented-3d-boxes", + "required_split": "validation", + "retuning_allowed": false + }, + "profile_id": "l3-pointpillars-public-transfer-probe-v1", + "runtime_policy": { + "engine_built_on_target_required": true, + "existing_triton_only": true, + "precision": "strongly-typed", + "second_serving_stack_allowed": false, + "triton_image": "nvcr.io/nvidia/tritonserver:26.06-py3", + "triton_image_digest": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" + }, + "schema_version": "missioncore.l3-pointpillars-benchmark-profile/v1" +} diff --git a/experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json b/experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json new file mode 100644 index 0000000..3d4f702 --- /dev/null +++ b/experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json @@ -0,0 +1,125 @@ +{ + "schema_version": "missioncore.l3-pointpillars-onnx-contract/v1", + "source": { + "model_id": "nvidia/tao/pointpillarnet", + "model_version": "deployable_v1.1", + "onnx_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1" + }, + "extraction": { + "onnx_version": "1.22.0", + "method": "read-only ONNX graph attribute inspection", + "observed_at_utc": "2026-07-31T07:16:00Z" + }, + "inputs": [ + { + "dtype": "FP32", + "name": "points", + "shape": [ + "batch", + 204800, + 4 + ] + }, + { + "dtype": "INT32", + "name": "num_points", + "shape": [ + "batch" + ] + } + ], + "outputs": [ + { + "dtype": "FP32", + "name": "output_boxes", + "shape": [ + "batch", + 393216, + 9 + ] + }, + { + "dtype": "INT32", + "name": "num_boxes", + "shape": [ + "batch" + ] + } + ], + "plugins": { + "DecodeBbox3DPlugin": { + "anchor_bottom_height": [ + -1.7799999713897705, + -0.6000000238418579, + -0.6000000238418579 + ], + "anchors": [ + 3.9000000953674316, + 1.600000023841858, + 1.559999942779541, + 0.0, + 3.9000000953674316, + 1.600000023841858, + 1.559999942779541, + 1.5700000524520874, + 0.800000011920929, + 0.6000000238418579, + 1.7300000190734863, + 0.0, + 0.800000011920929, + 0.6000000238418579, + 1.7300000190734863, + 1.5700000524520874, + 1.7599999904632568, + 0.6000000238418579, + 1.7300000190734863, + 0.0, + 1.7599999904632568, + 0.6000000238418579, + 1.7300000190734863, + 1.5700000524520874 + ], + "dir_limit_offset": 0.0, + "dir_offset": 0.7853900194168091, + "num_dir_bins": 2, + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432 + ], + "score_threshold": 0.10000000149011612 + }, + "PillarScatterPlugin": { + "dense_shape": [ + 512, + 512 + ] + }, + "VoxelGeneratorPlugin": { + "max_num_points_per_voxel": 32, + "max_voxels": 10000, + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432 + ], + "voxel_feature_num": 10, + "voxel_size": [ + 0.20000000298023224, + 0.20000000298023224, + 5.800000190734863 + ] + } + }, + "claim_boundary": { + "model_card_training_domain": "proprietary-solid-state-lidar", + "public_training_truth_available": false, + "public_kitti_is_cross_domain": true + } +} diff --git a/experiments/perception/l3_worker_inventory_2026-07-30.json b/experiments/perception/l3_worker_inventory_2026-07-30.json new file mode 100644 index 0000000..8bdbe87 --- /dev/null +++ b/experiments/perception/l3_worker_inventory_2026-07-30.json @@ -0,0 +1,163 @@ +{ + "gpu": { + "driver_version": "610.47", + "memory_total_mib": 24564, + "name": "NVIDIA GeForce RTX 4090" + }, + "host_id": "worker-006", + "observed_at_utc": "2026-07-31T06:50:46.660883Z", + "schema_version": "missioncore.l3-worker-inventory/v1", + "serving_stack_count": 1, + "staged_models": [ + { + "artifact_sha256": "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481", + "backend": "tensorrt", + "model_classes": [ + "Vehicle", + "Pedestrian", + "Cyclist" + ], + "engine_built_on_target": true, + "input_fields": [ + "x", + "y", + "z", + "intensity" + ], + "maximum_points": 204800, + "name": "pointpillars", + "outputs": [ + { + "dtype": "FP32", + "name": "output_boxes", + "shape": [ + 1, + 393216, + 9 + ] + }, + { + "dtype": "INT32", + "name": "num_boxes", + "shape": [ + 1 + ] + } + ], + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432 + ], + "precision": "strongly-typed", + "provenance_verified": true, + "representation_smoke": { + "accuracy_evaluated": false, + "input_artifact_sha256": "95be16aec8496260ec25ff716c79d343a3fd9e3f4957df99bb8007c6b21c9f59", + "input_point_count": 169883, + "navigation_or_safety_accepted": false, + "single_query_gpu_compute_ms": 55.2069, + "source_frame_id": "2022-07-22_flight__0071_1658494234334310308", + "source_id": "goose-3d/v2025-08-22", + "status": "engine-executed" + }, + "source_format": "onnx", + "source_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da", + "source_model_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1", + "upstream_version": "deployable_v1.1" + } + ], + "triton": { + "container_name": "ndc-mission-core-triton", + "healthy": true, + "image": "nvcr.io/nvidia/tritonserver:26.06-py3", + "image_digest": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "model_control_mode": "explicit", + "model_repository_read_only": true, + "models": [ + { + "artifact_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063", + "backend": "onnxruntime", + "name": "yolox_s" + }, + { + "artifact_sha256": "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481", + "backend": "tensorrt", + "engine_built_on_target": true, + "input_fields": [ + "x", + "y", + "z", + "intensity" + ], + "live_schema_smoke": { + "accuracy_evaluated": false, + "inference_ms": 106.79901000548853, + "input_artifact_sha256": "59a02fdaaab3b7e903713cb618e8f53efcaf71c144436ddfcdf4f28bdbd73d20", + "input_point_count": 120268, + "observed_model_classes": [ + "Cyclist", + "Vehicle" + ], + "output_box_count": 58, + "source_frame_id": "000001", + "source_id": "kitti-3d-object/v2017", + "status": "engine-schema-executed" + }, + "maximum_points": 204800, + "model_classes": [ + "Vehicle", + "Pedestrian", + "Cyclist" + ], + "name": "pointpillars", + "outputs": [ + { + "dtype": "FP32", + "name": "output_boxes", + "shape": [ + 1, + 393216, + 9 + ] + }, + { + "dtype": "INT32", + "name": "num_boxes", + "shape": [ + 1 + ] + } + ], + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432 + ], + "precision": "strongly-typed", + "provenance_verified": true, + "representation_smoke": { + "accuracy_evaluated": false, + "input_artifact_sha256": "95be16aec8496260ec25ff716c79d343a3fd9e3f4957df99bb8007c6b21c9f59", + "input_point_count": 169883, + "navigation_or_safety_accepted": false, + "single_query_gpu_compute_ms": 55.2069, + "source_frame_id": "2022-07-22_flight__0071_1658494234334310308", + "source_id": "goose-3d/v2025-08-22", + "status": "engine-executed" + }, + "source_format": "onnx", + "source_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da", + "source_model_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1", + "upstream_version": "deployable_v1.1" + } + ], + "strict_readiness": true + } +} diff --git a/experiments/perception/prepare_l3_pointpillars_worker_package.py b/experiments/perception/prepare_l3_pointpillars_worker_package.py new file mode 100644 index 0000000..5626965 --- /dev/null +++ b/experiments/perception/prepare_l3_pointpillars_worker_package.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Build the minimal immutable L3 PointPillars package for Worker 006.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +SCHEMA = "missioncore.l3-pointpillars-worker-package/v1" +_SOURCE_FILES = { + "runtime/k1link/__init__.py": "src/k1link/__init__.py", + "runtime/k1link/artifacts.py": "src/k1link/artifacts.py", + "runtime/k1link/compute/kitti_pointpillars_benchmark.py": ( + "src/k1link/compute/kitti_pointpillars_benchmark.py" + ), + "runtime/k1link/compute/l3_pointpillars_admission.py": ( + "src/k1link/compute/l3_pointpillars_admission.py" + ), + "runtime/k1link/compute/pointpillars_postprocess.py": ( + "src/k1link/compute/pointpillars_postprocess.py" + ), + "runtime/k1link/datasets/kitti_3d_admission.py": ( + "src/k1link/datasets/kitti_3d_admission.py" + ), + "runtime/install_l3_pointpillars_triton_model.py": ( + "experiments/perception/worker/install_l3_pointpillars_triton_model.py" + ), + "runtime/admit_l3_kitti_release.py": ( + "experiments/perception/worker/admit_l3_kitti_release.py" + ), + "runtime/run_l3_pointpillars_admission.py": ( + "experiments/perception/run_l3_pointpillars_admission.py" + ), + "runtime/run_l3_pointpillars_public_baseline.py": ( + "experiments/perception/worker/run_l3_pointpillars_public_baseline.py" + ), + "runtime/smoke_l3_pointpillars_live.py": ( + "experiments/perception/worker/smoke_l3_pointpillars_live.py" + ), +} +_GENERATED_FILES = { + "runtime/k1link/compute/__init__.py": ( + '"""Minimal L3 worker projection; import compute modules explicitly."""\n' + ), + "runtime/k1link/datasets/__init__.py": ( + '"""Minimal L3 worker projection; import KITTI admission explicitly."""\n' + ), +} +_INPUT_FILES = { + "input/profile.json": "experiments/perception/l3_pointpillars_benchmark_profile.json", + "input/onnx-contract.json": ( + "experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json" + ), + "input/dataset-inventory.json": ( + "experiments/perception/l3_dataset_inventory_2026-07-30.json" + ), + "input/worker-inventory.json": ( + "experiments/perception/l3_worker_inventory_2026-07-30.json" + ), + "input/triton/config.pbtxt": "experiments/perception/triton/pointpillars/config.pbtxt", +} + + +class L3WorkerPackageError(RuntimeError): + """The L3 worker package source or immutable package is invalid.""" + + +def build_l3_worker_package( + *, + repository_root: Path, + output_root: Path, + admission_result: Path | None = None, +) -> Path: + """Build or reopen a content-addressed minimal Worker 006 runtime.""" + + repository = repository_root.resolve(strict=True) + sources: dict[str, Path | None] = { + target: repository / relative for target, relative in _SOURCE_FILES.items() + } + sources.update( + { + target: repository / relative + for target, relative in _INPUT_FILES.items() + } + ) + sources.update({target: None for target in _GENERATED_FILES}) + if admission_result is not None: + result = admission_result.resolve(strict=True) + if not result.is_dir() or result.is_symlink(): + raise L3WorkerPackageError("L3 admission result root is invalid") + admission_root = f"input/admission/{result.name}" + sources[f"{admission_root}/manifest.json"] = result / "manifest.json" + sources[f"{admission_root}/admission-report.json"] = ( + result / "admission-report.json" + ) + + descriptors: list[dict[str, Any]] = [] + for relative, source in sorted(sources.items()): + payload = _payload(relative, source) + descriptors.append( + { + "path": relative, + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + ) + identity = { + "schema_version": SCHEMA, + "classification": "minimal-l3-pointpillars-worker-runtime", + "worker_host_id": "worker-006", + "source_artifacts": descriptors, + "artifact_paths": [row["path"] for row in descriptors], + "execution_policy": { + "sequential": True, + "parallel_workers": 1, + "existing_triton_only": True, + "container_creation_allowed": False, + "container_restart_allowed": False, + "raw_tensor_export_allowed": False, + }, + "authority": { + "shadow_only": True, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + package_id = f"l3-pointpillars-worker-package-{identity_sha256}" + output = output_root.expanduser().absolute() + output.mkdir(mode=0o700, parents=True, exist_ok=True) + destination = output / package_id + if destination.exists(): + validate_l3_worker_package(destination) + return destination + + staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp" + staging.mkdir(mode=0o700, exist_ok=False) + try: + for relative, source in sources.items(): + target = staging / relative + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + if source is None: + target.write_text(_GENERATED_FILES[relative], encoding="utf-8") + else: + shutil.copyfile(source, target) + artifacts = [ + { + "kind": relative, + "path": relative, + "byte_length": (staging / relative).stat().st_size, + "sha256": _sha256(staging / relative), + } + for relative in sorted(sources) + ] + manifest = { + "schema_version": SCHEMA, + "package_id": package_id, + "identity_sha256": identity_sha256, + "identity": identity, + "created_at_utc": datetime.now(UTC) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z"), + "artifacts": artifacts, + } + _write_json(staging / "manifest.json", manifest) + validate_l3_worker_package(staging, allow_staging=True) + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + validate_l3_worker_package(destination) + return destination + + +def validate_l3_worker_package( + root: Path, + *, + allow_staging: bool = False, +) -> dict[str, Any]: + """Validate package identity, exact file set, and every member digest.""" + + resolved = root.resolve(strict=True) + manifest = _read_json(resolved / "manifest.json") + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + package_id = manifest.get("package_id") + artifacts = manifest.get("artifacts") + expected_name = ( + isinstance(package_id, str) + and ( + resolved.name == package_id + or ( + allow_staging + and resolved.name.startswith(f".{package_id}.") + and resolved.name.endswith(".tmp") + ) + ) + ) + if ( + manifest.get("schema_version") != SCHEMA + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or package_id != f"l3-pointpillars-worker-package-{identity_sha256}" + or not expected_name + or not isinstance(artifacts, list) + ): + raise L3WorkerPackageError("L3 worker package identity is invalid") + expected_paths = set(identity.get("artifact_paths", [])) + actual_paths = { + path.relative_to(resolved).as_posix() + for path in resolved.rglob("*") + if path.is_file() + } + if ( + not expected_paths + or actual_paths != expected_paths | {"manifest.json"} + or len(artifacts) != len(expected_paths) + ): + raise L3WorkerPackageError("L3 worker package file set changed") + observed: set[str] = set() + for row in artifacts: + if not isinstance(row, dict): + raise L3WorkerPackageError("L3 worker package artifact is invalid") + relative = row.get("path") + path = resolved / str(relative) + if ( + not isinstance(relative, str) + or relative not in expected_paths + or relative in observed + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not path.is_file() + or path.is_symlink() + or row.get("kind") != relative + or row.get("byte_length") != path.stat().st_size + or row.get("sha256") != _sha256(path) + ): + raise L3WorkerPackageError("L3 worker package artifact changed") + observed.add(relative) + if observed != expected_paths: + raise L3WorkerPackageError("L3 worker package artifact coverage changed") + return manifest + + +def _payload(relative: str, source: Path | None) -> bytes: + if source is None: + return _GENERATED_FILES[relative].encode() + if not source.is_file() or source.is_symlink(): + raise L3WorkerPackageError(f"L3 package source is invalid: {relative}") + return source.read_bytes() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, json.JSONDecodeError) as exc: + raise L3WorkerPackageError("L3 worker package manifest is invalid") from exc + if not isinstance(value, dict): + raise L3WorkerPackageError("L3 worker package manifest is invalid") + return value + + +def _write_json(path: Path, value: object) -> None: + with path.open("x", encoding="utf-8", newline="\n") as stream: + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repository-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--admission-result", type=Path) + args = parser.parse_args() + package = build_l3_worker_package( + repository_root=args.repository_root, + output_root=args.output_root, + admission_result=args.admission_result, + ) + print(package) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_l3_pointpillars_admission.py b/experiments/perception/run_l3_pointpillars_admission.py new file mode 100644 index 0000000..c3f3f72 --- /dev/null +++ b/experiments/perception/run_l3_pointpillars_admission.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Build the immutable L3 NVIDIA PointPillars benchmark admission.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.compute.l3_pointpillars_admission import ( + build_l3_pointpillars_admission, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--dataset-inventory", type=Path, required=True) + parser.add_argument("--worker-inventory", type=Path, required=True) + parser.add_argument( + "--output-root", + type=Path, + default=Path(".runtime/compute-experiments/l3/pointpillars-admissions"), + ) + args = parser.parse_args() + result = build_l3_pointpillars_admission( + profile_path=args.profile, + dataset_inventory_path=args.dataset_inventory, + worker_inventory_path=args.worker_inventory, + output_root=args.output_root, + ) + print( + json.dumps( + { + "result_id": result.result_id, + "result_root": str(result.result_root), + "status": result.report["status"], + "blocker_codes": result.report["blocker_codes"], + "next_gate": result.report["next_gate"], + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 if result.public_transfer_probe_authorized else 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/triton/pointpillars/config.pbtxt b/experiments/perception/triton/pointpillars/config.pbtxt new file mode 100644 index 0000000..3c82091 --- /dev/null +++ b/experiments/perception/triton/pointpillars/config.pbtxt @@ -0,0 +1,42 @@ +name: "pointpillars" +platform: "tensorrt_plan" +max_batch_size: 0 + +input [ + { + name: "points" + data_type: TYPE_FP32 + dims: [1, 204800, 4] + }, + { + name: "num_points" + data_type: TYPE_INT32 + dims: [1] + } +] + +output [ + { + name: "output_boxes" + data_type: TYPE_FP32 + dims: [1, 393216, 9] + }, + { + name: "num_boxes" + data_type: TYPE_INT32 + dims: [1] + } +] + +instance_group [ + { + count: 1 + kind: KIND_GPU + } +] + +version_policy { + specific { + versions: 1 + } +} diff --git a/experiments/perception/worker/admit_l3_kitti_release.py b/experiments/perception/worker/admit_l3_kitti_release.py new file mode 100644 index 0000000..5459c5e --- /dev/null +++ b/experiments/perception/worker/admit_l3_kitti_release.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Admit the canonical archive-only KITTI release on Worker 006.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from k1link.datasets.kitti_3d_admission import ( + admit_kitti_3d_object_release, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset-root", + type=Path, + default=Path("/mnt/d/NDC_MISSIONCORE/datasets"), + ) + args = parser.parse_args() + result = admit_kitti_3d_object_release(args.dataset_root) + print( + json.dumps( + { + "source_id": result["source_id"], + "status": result["status"], + "release_identity_sha256": result["release_identity_sha256"], + "training_frame_count": result["alignment"][ + "training_frame_count" + ], + "validation_frame_count": result["alignment"]["split_counts"][ + "validation" + ], + "next_action": result["next_action"], + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/install_l3_pointpillars_triton_model.py b/experiments/perception/worker/install_l3_pointpillars_triton_model.py new file mode 100644 index 0000000..d6f9045 --- /dev/null +++ b/experiments/perception/worker/install_l3_pointpillars_triton_model.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Promote the verified PointPillars engine into canonical Triton exactly once.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import tempfile +import urllib.request +from contextlib import suppress +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from k1link.compute.l3_pointpillars_admission import ( + read_l3_pointpillars_admission, +) + +PROVENANCE_SCHEMA = "missioncore.triton-model-provenance/v1" +MODEL_NAME = "pointpillars" +EXPECTED_ENGINE_SHA256 = ( + "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481" +) +EXPECTED_ENGINE_BYTES = 8_785_436 +EXPECTED_CONFIG_SHA256 = ( + "a68e7e37ae611b7d4c3fb633b31360ddbdf26ab0a37a363cc566357e614a385c" +) +EXPECTED_SOURCE_SHA256 = ( + "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1" +) +EXPECTED_LABEL_SHA256 = ( + "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da" +) +CANONICAL_MODEL_REPOSITORY = Path("/mnt/d/NDC_MISSIONCORE/runtime/models") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--admission-result", type=Path, required=True) + parser.add_argument("--engine", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--model-repository", + type=Path, + default=CANONICAL_MODEL_REPOSITORY, + ) + parser.add_argument("--triton-url", default="http://127.0.0.1:8000") + args = parser.parse_args() + + admission = read_l3_pointpillars_admission(args.admission_result) + if ( + admission.report.get("status") != "blocked-foundation-assets" + or admission.report.get("blocker_codes") + != ["pointpillars-model-not-installed-live"] + or admission.report.get("eligible_public_probe_dataset_ids") + != ["kitti-3d-object/v2017"] + or admission.report.get("detector", {}).get("staged_target_engine_ready") + is not True + ): + raise RuntimeError( + "L3 admission does not authorize canonical PointPillars promotion" + ) + engine = args.engine.resolve(strict=True) + config = args.config.resolve(strict=True) + if engine.stat().st_size != EXPECTED_ENGINE_BYTES or _sha256( + engine + ) != EXPECTED_ENGINE_SHA256: + raise RuntimeError("PointPillars target engine identity changed") + if _sha256(config) != EXPECTED_CONFIG_SHA256: + raise RuntimeError("PointPillars Triton configuration identity changed") + repository = args.model_repository.expanduser().absolute() + if repository != CANONICAL_MODEL_REPOSITORY or not repository.is_dir(): + raise RuntimeError("PointPillars promotion requires the canonical model repository") + destination = repository / MODEL_NAME + if destination.exists(): + _validate_existing(destination) + _require_model_ready(args.triton_url) + print( + json.dumps( + {"status": "already-installed", "model": MODEL_NAME}, + sort_keys=True, + ) + ) + return 0 + + provenance = { + "schema_version": PROVENANCE_SCHEMA, + "model_name": MODEL_NAME, + "upstream_model_id": "nvidia/tao/pointpillarnet", + "upstream_version": "deployable_v1.1", + "source_format": "onnx", + "source_model_sha256": EXPECTED_SOURCE_SHA256, + "source_label_sha256": EXPECTED_LABEL_SHA256, + "engine_sha256": EXPECTED_ENGINE_SHA256, + "engine_byte_length": EXPECTED_ENGINE_BYTES, + "config_sha256": EXPECTED_CONFIG_SHA256, + "engine_built_on_target": True, + "worker_host_id": "worker-006", + "gpu": "NVIDIA GeForce RTX 4090", + "compute_capability": "8.9", + "tensorrt_version": "11.0.0", + "precision": "strongly-typed", + "admission_result_id": admission.result_id, + "installed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "authority": { + "shadow_only": True, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + staging = Path( + tempfile.mkdtemp( + dir=repository, + prefix=f".{MODEL_NAME}.", + suffix=".incomplete", + ) + ) + try: + version = staging / "1" + version.mkdir(mode=0o700) + shutil.copyfile(engine, version / "model.plan") + shutil.copyfile(config, staging / "config.pbtxt") + _atomic_json(staging / "provenance.json", provenance) + if ( + _sha256(version / "model.plan") != EXPECTED_ENGINE_SHA256 + or _sha256(staging / "config.pbtxt") != EXPECTED_CONFIG_SHA256 + ): + raise RuntimeError("PointPillars staged model package changed during copy") + os.replace(staging, destination) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + request = urllib.request.Request( + f"{args.triton_url.rstrip('/')}/v2/repository/models/{MODEL_NAME}/load", + data=b"{}", + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + if response.status != 200: + raise RuntimeError("canonical Triton rejected PointPillars load") + _require_model_ready(args.triton_url) + except OSError as exc: + raise RuntimeError( + "PointPillars package is installed but canonical Triton did not load it" + ) from exc + print( + json.dumps( + { + "status": "installed-and-ready", + "model": MODEL_NAME, + "engine_sha256": EXPECTED_ENGINE_SHA256, + "provenance_sha256": _sha256(destination / "provenance.json"), + }, + sort_keys=True, + ) + ) + return 0 + + +def _validate_existing(destination: Path) -> None: + engine = destination / "1/model.plan" + config = destination / "config.pbtxt" + provenance_path = destination / "provenance.json" + if ( + not engine.is_file() + or not config.is_file() + or not provenance_path.is_file() + or engine.stat().st_size != EXPECTED_ENGINE_BYTES + or _sha256(engine) != EXPECTED_ENGINE_SHA256 + or _sha256(config) != EXPECTED_CONFIG_SHA256 + ): + raise RuntimeError("existing PointPillars model package is not admitted") + try: + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError("existing PointPillars provenance is invalid") from exc + if ( + not isinstance(provenance, dict) + or provenance.get("schema_version") != PROVENANCE_SCHEMA + or provenance.get("model_name") != MODEL_NAME + or provenance.get("engine_sha256") != EXPECTED_ENGINE_SHA256 + or provenance.get("source_model_sha256") != EXPECTED_SOURCE_SHA256 + or provenance.get("source_label_sha256") != EXPECTED_LABEL_SHA256 + or provenance.get("engine_built_on_target") is not True + ): + raise RuntimeError("existing PointPillars provenance is invalid") + + +def _require_model_ready(url: str) -> None: + request = urllib.request.Request( + f"{url.rstrip('/')}/v2/models/{MODEL_NAME}/ready", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status != 200: + raise RuntimeError("PointPillars is not ready") + except OSError as exc: + raise RuntimeError("PointPillars is not ready in canonical Triton") from exc + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024**2), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(payload: Any) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + descriptor, temporary = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + try: + with os.fdopen(descriptor, "wb") as target: + target.write(_canonical_json(payload) + b"\n") + target.flush() + os.fsync(target.fileno()) + os.replace(temporary, path) + except BaseException: + with suppress(OSError): + os.unlink(temporary) + raise + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/run_l3_pointpillars_public_baseline.py b/experiments/perception/worker/run_l3_pointpillars_public_baseline.py new file mode 100644 index 0000000..35b401b --- /dev/null +++ b/experiments/perception/worker/run_l3_pointpillars_public_baseline.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""Run the complete KITTI validation baseline through canonical Triton.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import tempfile +import time +import urllib.request +import zipfile +from contextlib import suppress +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import numpy as np + +from k1link.compute.kitti_pointpillars_benchmark import ( + PointPillarsFramePrediction, + evaluate_pointpillars_predictions, + read_kitti_validation_truth, +) +from k1link.compute.pointpillars_postprocess import ( + POINTPILLARS_EMBEDDED_SCORE_THRESHOLD, + POINTPILLARS_MODEL_POINT_CLOUD_RANGE, + PointPillarsBox, + decode_pointpillars_output, +) +from k1link.datasets.kitti_3d_admission import ( + KITTI_3D_RELEASE_ROOT, + KITTI_CALIB_ARCHIVE, + KITTI_LABEL_ARCHIVE, + KITTI_VELODYNE_ARCHIVE, + read_kitti_3d_admission, + read_kitti_standard_splits, +) + +RUN_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-run/v1" +FRAME_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-frame/v1" +REPORT_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-report/v1" +MANIFEST_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-result/v1" +WORKER_PACKAGE_SCHEMA = "missioncore.l3-pointpillars-worker-package/v1" +MODEL_NAME = "pointpillars" +MAXIMUM_POINTS = 204_800 +EXPECTED_MODEL_SHA256 = ( + "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1" +) +EXPECTED_LABEL_SHA256 = ( + "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da" +) +EXPECTED_ENGINE_SHA256 = ( + "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481" +) +EXPECTED_ONNX_CONTRACT_SHA256 = ( + "2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be" +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--model-provenance", type=Path, required=True) + parser.add_argument("--worker-package", type=Path, required=True) + parser.add_argument( + "--output-root", + type=Path, + default=Path( + "/mnt/d/NDC_MISSIONCORE/runtime/experiments/l3/pointpillars-kitti" + ), + ) + parser.add_argument("--triton-url", default="http://127.0.0.1:8000") + args = parser.parse_args() + + dataset_root = args.dataset_root.expanduser().absolute() + admission = read_kitti_3d_admission(dataset_root) + splits = read_kitti_standard_splits(dataset_root) + validation_frame_ids = splits["validation"] + profile_path = args.profile.resolve(strict=True) + provenance_path = args.model_provenance.resolve(strict=True) + profile = _read_json(profile_path) + provenance = _read_json(provenance_path) + _validate_profile_and_provenance(profile, provenance) + worker_package = _read_worker_package(args.worker_package) + _require_triton_ready(args.triton_url) + + identity = { + "schema_version": RUN_SCHEMA, + "dataset_source_id": admission["source_id"], + "dataset_release_identity_sha256": admission["release_identity_sha256"], + "validation_frame_count": len(validation_frame_ids), + "validation_frame_ids_sha256": hashlib.sha256( + "\n".join(validation_frame_ids).encode("ascii") + b"\n" + ).hexdigest(), + "profile_sha256": _sha256(profile_path), + "model_provenance_sha256": _sha256(provenance_path), + "worker_package_id": worker_package["package_id"], + "worker_package_identity_sha256": worker_package["identity_sha256"], + "engine_sha256": EXPECTED_ENGINE_SHA256, + "producer_sha256": _sha256(Path(__file__)), + "execution": { + "worker_host_id": "worker-006", + "triton_model_name": MODEL_NAME, + "sequential": True, + "parallel_workers": 1, + }, + "authority": { + "shadow_only": True, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + run_id = f"l3-pointpillars-kitti-{identity_sha256}" + run_root = args.output_root.expanduser().absolute() / run_id + frames_root = run_root / "frames" + frames_root.mkdir(mode=0o700, parents=True, exist_ok=True) + _write_once(run_root / "identity.json", identity) + + archive_root = dataset_root / KITTI_3D_RELEASE_ROOT / "archives" + points_archive = archive_root / KITTI_VELODYNE_ARCHIVE + labels_archive = archive_root / KITTI_LABEL_ARCHIVE + calibrations_archive = archive_root / KITTI_CALIB_ARCHIVE + truth_by_frame = read_kitti_validation_truth( + labels_archive=labels_archive, + calibrations_archive=calibrations_archive, + validation_frame_ids=validation_frame_ids, + ) + + predictions: list[PointPillarsFramePrediction] = [] + try: + with zipfile.ZipFile(points_archive.resolve(strict=True)) as points_zip: + for completed, frame_id in enumerate(validation_frame_ids, start=1): + member = f"training/velodyne/{frame_id}.bin" + point_bytes = points_zip.read(member) + point_sha256 = hashlib.sha256(point_bytes).hexdigest() + frame_path = frames_root / f"{frame_id}.json" + if frame_path.exists(): + prediction = _read_frame( + frame_path, + frame_id=frame_id, + point_sha256=point_sha256, + ) + else: + prediction = _run_frame( + triton_url=args.triton_url, + frame_id=frame_id, + point_bytes=point_bytes, + ) + _atomic_json( + frame_path, + _frame_payload(prediction, point_sha256=point_sha256), + ) + predictions.append(prediction) + if completed == 1 or completed % 50 == 0: + print( + json.dumps( + { + "run_id": run_id, + "completed": completed, + "total": len(validation_frame_ids), + "frame_id": frame_id, + }, + sort_keys=True, + ), + flush=True, + ) + except (OSError, KeyError, zipfile.BadZipFile) as exc: + raise RuntimeError("KITTI validation point clouds could not be read") from exc + + metrics = evaluate_pointpillars_predictions( + truth_by_frame=truth_by_frame, + predictions=tuple(predictions), + ) + report = { + "schema_version": REPORT_SCHEMA, + "run_id": run_id, + "status": "public-cross-domain-transfer-probe-measured", + "dataset_source_id": admission["source_id"], + "dataset_release_identity_sha256": admission["release_identity_sha256"], + "model": { + "name": MODEL_NAME, + "source_model_sha256": EXPECTED_MODEL_SHA256, + "source_label_sha256": EXPECTED_LABEL_SHA256, + "engine_sha256": EXPECTED_ENGINE_SHA256, + }, + "metrics": metrics, + "completed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "authority": identity["authority"], + } + _atomic_json(run_root / "report.json", report) + manifest = { + "schema_version": MANIFEST_SCHEMA, + "run_id": run_id, + "identity_sha256": identity_sha256, + "identity": identity, + "status": report["status"], + "artifacts": [ + _artifact(run_root / "identity.json", "run-identity"), + _artifact(run_root / "report.json", "benchmark-report"), + ], + "frame_result_count": len(predictions), + "frame_results_identity_sha256": _frame_results_identity(frames_root), + "authority": identity["authority"], + } + _atomic_json(run_root / "manifest.json", manifest) + print( + json.dumps( + { + "run_id": run_id, + "status": report["status"], + "frame_count": len(predictions), + "bev_map40": metrics["aggregates"]["bev_map40"], + "3d_map40": metrics["aggregates"]["3d_map40"], + "false_occupied_rate": metrics["aggregates"][ + "false_occupied_rate" + ], + }, + sort_keys=True, + ), + flush=True, + ) + return 0 + + +def _run_frame( + *, + triton_url: str, + frame_id: str, + point_bytes: bytes, +) -> PointPillarsFramePrediction: + if not point_bytes or len(point_bytes) % 16: + raise RuntimeError("KITTI point frame is not packed XYZI") + native = np.frombuffer(point_bytes, dtype=" MAXIMUM_POINTS: + raise RuntimeError("KITTI point frame violates the model input contract") + points = np.zeros((1, MAXIMUM_POINTS, 4), dtype=np.float32) + points[0, : native.shape[0]] = native + num_points = np.asarray([native.shape[0]], dtype=np.int32) + output_boxes, output_count, elapsed_ms = _infer( + triton_url, + points, + num_points, + ) + boxes = decode_pointpillars_output(output_boxes, output_count) + return PointPillarsFramePrediction( + frame_id=frame_id, + boxes=boxes, + inference_ms=elapsed_ms, + ) + + +def _infer( + triton_url: str, + points: np.ndarray, + num_points: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, float]: + points_binary = np.ascontiguousarray(points, dtype=np.float32).tobytes() + count_binary = np.ascontiguousarray(num_points, dtype=np.int32).tobytes() + header = { + "inputs": [ + { + "name": "points", + "shape": [1, MAXIMUM_POINTS, 4], + "datatype": "FP32", + "parameters": {"binary_data_size": len(points_binary)}, + }, + { + "name": "num_points", + "shape": [1], + "datatype": "INT32", + "parameters": {"binary_data_size": len(count_binary)}, + }, + ], + "outputs": [ + {"name": "output_boxes", "parameters": {"binary_data": True}}, + {"name": "num_boxes", "parameters": {"binary_data": True}}, + ], + } + encoded_header = _canonical_json(header) + request = urllib.request.Request( + f"{triton_url.rstrip('/')}/v2/models/{MODEL_NAME}/infer", + data=encoded_header + points_binary + count_binary, + headers={ + "Content-Type": "application/octet-stream", + "Inference-Header-Content-Length": str(len(encoded_header)), + }, + method="POST", + ) + started = time.perf_counter() + with urllib.request.urlopen(request, timeout=120) as response: + payload = response.read() + header_length = int(response.headers["Inference-Header-Content-Length"]) + elapsed_ms = (time.perf_counter() - started) * 1000.0 + response_header = json.loads(payload[:header_length]) + outputs = response_header.get("outputs") + if not isinstance(outputs, list) or len(outputs) != 2: + raise RuntimeError("Triton PointPillars output set changed") + offset = header_length + decoded: dict[str, np.ndarray] = {} + for descriptor in outputs: + if not isinstance(descriptor, dict): + raise RuntimeError("Triton PointPillars output descriptor is invalid") + name = descriptor.get("name") + parameters = descriptor.get("parameters") + if not isinstance(name, str) or not isinstance(parameters, dict): + raise RuntimeError("Triton PointPillars output descriptor is invalid") + byte_length = parameters.get("binary_data_size") + if ( + isinstance(byte_length, bool) + or not isinstance(byte_length, int) + or byte_length < 1 + ): + raise RuntimeError("Triton PointPillars binary output size is invalid") + binary = payload[offset : offset + byte_length] + if len(binary) != byte_length: + raise RuntimeError("Triton PointPillars binary output is truncated") + offset += byte_length + if name == "output_boxes" and descriptor.get("datatype") == "FP32": + decoded[name] = np.frombuffer(binary, dtype=" None: + for endpoint in ("/v2/health/ready", f"/v2/models/{MODEL_NAME}/ready"): + request = urllib.request.Request(f"{url.rstrip('/')}{endpoint}", method="GET") + try: + with urllib.request.urlopen(request, timeout=10) as response: + if response.status != 200: + raise RuntimeError("canonical Triton is not ready") + except OSError as exc: + raise RuntimeError("canonical Triton or PointPillars is not ready") from exc + + +def _validate_profile_and_provenance( + profile: dict[str, Any], + provenance: dict[str, Any], +) -> None: + detector = profile.get("detector") + postprocessing = ( + detector.get("postprocessing") if isinstance(detector, dict) else None + ) + if ( + profile.get("schema_version") + != "missioncore.l3-pointpillars-benchmark-profile/v1" + or not isinstance(detector, dict) + or not isinstance(postprocessing, dict) + or detector.get("candidate_source_sha256") != EXPECTED_MODEL_SHA256 + or detector.get("candidate_label_sha256") != EXPECTED_LABEL_SHA256 + or detector.get("triton_model_name") != MODEL_NAME + or detector.get("model_classes") + != ["Vehicle", "Pedestrian", "Cyclist"] + or detector.get("point_cloud_range") + != list(POINTPILLARS_MODEL_POINT_CLOUD_RANGE) + or detector.get("training_domain") + != "proprietary-solid-state-lidar" + or detector.get("training_ground_truth_publicly_reproducible") is not False + or detector.get("onnx_contract_sha256") + != EXPECTED_ONNX_CONTRACT_SHA256 + or postprocessing.get("embedded_score_threshold") + != POINTPILLARS_EMBEDDED_SCORE_THRESHOLD + or postprocessing.get("embedded_contract_source") + != "onnx-node-attributes" + ): + raise RuntimeError("L3 PointPillars profile is invalid") + if ( + provenance.get("model_name") != MODEL_NAME + or provenance.get("source_model_sha256") != EXPECTED_MODEL_SHA256 + or provenance.get("source_label_sha256") != EXPECTED_LABEL_SHA256 + or provenance.get("engine_sha256") != EXPECTED_ENGINE_SHA256 + or provenance.get("engine_built_on_target") is not True + or provenance.get("worker_host_id") != "worker-006" + ): + raise RuntimeError("live PointPillars provenance is invalid") + + +def _read_worker_package(path: Path) -> dict[str, Any]: + package_root = path.expanduser().resolve(strict=True) + runtime_package_root = Path(__file__).resolve(strict=True).parents[1] + if ( + not package_root.is_dir() + or package_root != runtime_package_root + or package_root.is_symlink() + ): + raise RuntimeError("L3 runner is not executing from its declared worker package") + manifest = _read_json(package_root / "manifest.json") + identity = manifest.get("identity") + identity_sha256 = manifest.get("identity_sha256") + package_id = manifest.get("package_id") + artifacts = manifest.get("artifacts") + if ( + manifest.get("schema_version") != WORKER_PACKAGE_SCHEMA + or not isinstance(identity, dict) + or not isinstance(identity_sha256, str) + or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 + or package_id != f"l3-pointpillars-worker-package-{identity_sha256}" + or package_root.name != package_id + or not isinstance(artifacts, list) + ): + raise RuntimeError("L3 worker package identity is invalid") + expected_paths = identity.get("artifact_paths") + if not isinstance(expected_paths, list) or not expected_paths: + raise RuntimeError("L3 worker package file set is invalid") + expected = set(expected_paths) + actual = { + member.relative_to(package_root).as_posix() + for member in package_root.rglob("*") + if member.is_file() + } + if actual != expected | {"manifest.json"} or len(artifacts) != len(expected): + raise RuntimeError("L3 worker package file set changed") + observed: set[str] = set() + for descriptor in artifacts: + if not isinstance(descriptor, dict): + raise RuntimeError("L3 worker package artifact is invalid") + relative = descriptor.get("path") + artifact = package_root / str(relative) + if ( + not isinstance(relative, str) + or relative not in expected + or relative in observed + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not artifact.is_file() + or artifact.is_symlink() + or descriptor.get("kind") != relative + or descriptor.get("byte_length") != artifact.stat().st_size + or descriptor.get("sha256") != _sha256(artifact) + ): + raise RuntimeError("L3 worker package artifact changed") + observed.add(relative) + if observed != expected: + raise RuntimeError("L3 worker package artifact coverage changed") + return manifest + + +def _frame_payload( + prediction: PointPillarsFramePrediction, + *, + point_sha256: str, +) -> dict[str, Any]: + return { + "schema_version": FRAME_SCHEMA, + "frame_id": prediction.frame_id, + "point_sha256": point_sha256, + "inference_ms": prediction.inference_ms, + "boxes": [ + { + "x_m": box.x_m, + "y_m": box.y_m, + "z_m": box.z_m, + "length_m": box.length_m, + "width_m": box.width_m, + "height_m": box.height_m, + "yaw_rad": box.yaw_rad, + "class_id": box.class_id, + "model_class": box.model_class, + "score": box.score, + } + for box in prediction.boxes + ], + } + + +def _read_frame( + path: Path, + *, + frame_id: str, + point_sha256: str, +) -> PointPillarsFramePrediction: + payload = _read_json(path) + boxes = payload.get("boxes") + if ( + payload.get("schema_version") != FRAME_SCHEMA + or payload.get("frame_id") != frame_id + or payload.get("point_sha256") != point_sha256 + or not isinstance(boxes, list) + ): + raise RuntimeError("cached PointPillars frame result is invalid") + try: + decoded = tuple(PointPillarsBox(**box) for box in boxes) + inference_ms = float(payload["inference_ms"]) + except (KeyError, TypeError, ValueError) as exc: + raise RuntimeError("cached PointPillars frame result is invalid") from exc + if not math.isfinite(inference_ms) or inference_ms <= 0.0: + raise RuntimeError("cached PointPillars frame latency is invalid") + return PointPillarsFramePrediction( + frame_id=frame_id, + boxes=decoded, + inference_ms=inference_ms, + ) + + +def _frame_results_identity(frames_root: Path) -> str: + descriptors = [ + { + "name": path.name, + "sha256": _sha256(path), + "byte_length": path.stat().st_size, + } + for path in sorted(frames_root.glob("*.json")) + ] + return hashlib.sha256(_canonical_json(descriptors)).hexdigest() + + +def _artifact(path: Path, role: str) -> dict[str, Any]: + return { + "path": path.name, + "role": role, + "media_type": "application/json", + "sha256": _sha256(path), + "byte_length": path.stat().st_size, + } + + +def _read_json(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError(f"{path.name} is invalid") from exc + if not isinstance(payload, dict): + raise RuntimeError(f"{path.name} is not an object") + return payload + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024**2), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(payload: Any) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _write_once(path: Path, payload: dict[str, Any]) -> None: + encoded = _canonical_json(payload) + b"\n" + if path.exists(): + if path.read_bytes() != encoded: + raise RuntimeError(f"{path.name} identity changed") + return + _atomic_bytes(path, encoded) + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + _atomic_bytes(path, _canonical_json(payload) + b"\n") + + +def _atomic_bytes(path: Path, payload: bytes) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + try: + with os.fdopen(descriptor, "wb") as target: + target.write(payload) + target.flush() + os.fsync(target.fileno()) + os.replace(temporary, path) + except BaseException: + with suppress(OSError): + os.unlink(temporary) + raise + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/smoke_l3_pointpillars_live.py b/experiments/perception/worker/smoke_l3_pointpillars_live.py new file mode 100644 index 0000000..71378c1 --- /dev/null +++ b/experiments/perception/worker/smoke_l3_pointpillars_live.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Execute one schema-only PointPillars smoke frame in canonical Triton.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import zipfile +from datetime import UTC, datetime +from pathlib import Path + +from run_l3_pointpillars_public_baseline import ( + EXPECTED_ENGINE_SHA256, + EXPECTED_LABEL_SHA256, + EXPECTED_MODEL_SHA256, + _atomic_json, + _read_json, + _require_triton_ready, + _run_frame, + _validate_profile_and_provenance, +) + +from k1link.datasets.kitti_3d_admission import ( + KITTI_3D_RELEASE_ROOT, + KITTI_VELODYNE_ARCHIVE, + read_kitti_3d_admission, + read_kitti_standard_splits, +) + +SCHEMA = "missioncore.l3-pointpillars-live-smoke/v1" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--profile", type=Path, required=True) + parser.add_argument("--model-provenance", type=Path, required=True) + parser.add_argument("--frame-id") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--triton-url", default="http://127.0.0.1:8000") + args = parser.parse_args() + + dataset_root = args.dataset_root.expanduser().absolute() + admission = read_kitti_3d_admission(dataset_root) + validation = read_kitti_standard_splits(dataset_root)["validation"] + frame_id = args.frame_id or validation[0] + if frame_id not in validation: + raise RuntimeError("smoke frame is not in the admitted validation split") + profile = _read_json(args.profile.resolve(strict=True)) + provenance = _read_json(args.model_provenance.resolve(strict=True)) + _validate_profile_and_provenance(profile, provenance) + _require_triton_ready(args.triton_url) + + points_archive = ( + dataset_root + / KITTI_3D_RELEASE_ROOT + / "archives" + / KITTI_VELODYNE_ARCHIVE + ) + member = f"training/velodyne/{frame_id}.bin" + try: + with zipfile.ZipFile(points_archive.resolve(strict=True)) as source: + point_bytes = source.read(member) + except (OSError, KeyError, zipfile.BadZipFile) as exc: + raise RuntimeError("smoke point frame could not be read") from exc + prediction = _run_frame( + triton_url=args.triton_url, + frame_id=frame_id, + point_bytes=point_bytes, + ) + result = { + "schema_version": SCHEMA, + "status": "engine-schema-executed", + "source_id": admission["source_id"], + "dataset_release_identity_sha256": admission[ + "release_identity_sha256" + ], + "frame_id": frame_id, + "point_frame_sha256": hashlib.sha256(point_bytes).hexdigest(), + "point_count": len(point_bytes) // 16, + "post_nms_box_count": len(prediction.boxes), + "observed_model_classes": sorted( + {box.model_class for box in prediction.boxes} + ), + "inference_ms": prediction.inference_ms, + "model": { + "name": "pointpillars", + "source_model_sha256": EXPECTED_MODEL_SHA256, + "source_label_sha256": EXPECTED_LABEL_SHA256, + "engine_sha256": EXPECTED_ENGINE_SHA256, + }, + "observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "claim_boundary": { + "accuracy_measured": False, + "k1_transfer_evaluated": False, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + _atomic_json(args.output.expanduser().absolute(), result) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/k1link/compute/kitti_pointpillars_benchmark.py b/src/k1link/compute/kitti_pointpillars_benchmark.py new file mode 100644 index 0000000..7265df3 --- /dev/null +++ b/src/k1link/compute/kitti_pointpillars_benchmark.py @@ -0,0 +1,566 @@ +"""KITTI-to-LiDAR truth conversion and bounded PointPillars metrics.""" + +from __future__ import annotations + +import math +import zipfile +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +import numpy as np + +from k1link.compute.pointpillars_postprocess import ( + POINTPILLARS_MODEL_POINT_CLOUD_RANGE, + PointPillarsBox, + oriented_3d_iou, + oriented_bev_iou, +) + +KITTI_BENCHMARK_CLASSES: Final = ("Car", "Pedestrian", "Cyclist") +MODEL_TO_KITTI_CLASS: Final = { + "Vehicle": "Car", + "Pedestrian": "Pedestrian", + "Cyclist": "Cyclist", +} +KITTI_IOU_THRESHOLDS: Final = { + "Car": 0.7, + "Pedestrian": 0.5, + "Cyclist": 0.5, +} +KITTI_REFERENCE_POINT_CLOUD_RANGE: Final = ( + 0.0, + -39.68, + -3.0, + 69.12, + 39.68, + 1.0, +) +CROSS_DOMAIN_EVALUATION_RANGE: Final = tuple( + max( + KITTI_REFERENCE_POINT_CLOUD_RANGE[index], + POINTPILLARS_MODEL_POINT_CLOUD_RANGE[index], + ) + if index < 3 + else min( + KITTI_REFERENCE_POINT_CLOUD_RANGE[index], + POINTPILLARS_MODEL_POINT_CLOUD_RANGE[index], + ) + for index in range(6) +) +DISTANCE_BUCKETS_M: Final = ((0.0, 20.0), (20.0, 40.0), (40.0, 70.0)) + + +class KittiPointPillarsBenchmarkError(RuntimeError): + """KITTI truth or PointPillars predictions violate the frozen contract.""" + + +@dataclass(frozen=True, slots=True) +class KittiLidarTruth: + frame_id: str + benchmark_class: str + x_m: float + y_m: float + z_m: float + length_m: float + width_m: float + height_m: float + yaw_rad: float + + +@dataclass(frozen=True, slots=True) +class PointPillarsFramePrediction: + frame_id: str + boxes: tuple[PointPillarsBox, ...] + inference_ms: float + + +@dataclass(frozen=True, slots=True) +class _ScoredPrediction: + frame_id: str + index: int + benchmark_class: str + box: PointPillarsBox + + +@dataclass(frozen=True, slots=True) +class _MetricEvaluation: + average_precision_40: float + precision: float + recall: float + true_positives: int + false_positives: int + ground_truth_count: int + matched_truth: frozenset[tuple[str, int]] + matched_pairs: tuple[tuple[PointPillarsBox, KittiLidarTruth], ...] + + +def read_kitti_validation_truth( + *, + labels_archive: Path, + calibrations_archive: Path, + validation_frame_ids: tuple[str, ...], +) -> dict[str, tuple[KittiLidarTruth, ...]]: + """Read target cuboids and convert camera-bottom centers into LiDAR centers.""" + + if ( + not validation_frame_ids + or len(set(validation_frame_ids)) != len(validation_frame_ids) + or any(len(frame_id) != 6 or not frame_id.isdigit() for frame_id in validation_frame_ids) + ): + raise KittiPointPillarsBenchmarkError("KITTI validation frame index is invalid") + try: + with ( + zipfile.ZipFile(labels_archive.resolve(strict=True)) as labels_zip, + zipfile.ZipFile(calibrations_archive.resolve(strict=True)) as calib_zip, + ): + truth = { + frame_id: _read_frame_truth(labels_zip, calib_zip, frame_id) + for frame_id in validation_frame_ids + } + except (OSError, KeyError, UnicodeDecodeError, zipfile.BadZipFile) as exc: + raise KittiPointPillarsBenchmarkError( + "KITTI validation truth could not be read" + ) from exc + if any( + not any( + box.benchmark_class == class_name + for boxes in truth.values() + for box in boxes + ) + for class_name in KITTI_BENCHMARK_CLASSES + ): + raise KittiPointPillarsBenchmarkError("KITTI validation lacks a target class") + return truth + + +def evaluate_pointpillars_predictions( + *, + truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]], + predictions: tuple[PointPillarsFramePrediction, ...], +) -> dict[str, Any]: + """Evaluate the frozen model-to-KITTI mapping without retuning.""" + + if not truth_by_frame or set(truth_by_frame) != { + prediction.frame_id for prediction in predictions + }: + raise KittiPointPillarsBenchmarkError( + "prediction frames do not equal the admitted validation split" + ) + if len(predictions) != len(truth_by_frame): + raise KittiPointPillarsBenchmarkError("prediction frames contain duplicates") + if any( + not math.isfinite(prediction.inference_ms) or prediction.inference_ms <= 0.0 + for prediction in predictions + ): + raise KittiPointPillarsBenchmarkError("inference latency is invalid") + + all_boxes = tuple( + box + for prediction in predictions + for box in prediction.boxes + ) + evaluated_boxes = tuple( + box for box in all_boxes if _center_in_evaluation_range(box.x_m, box.y_m, box.z_m) + ) + scored = tuple( + _ScoredPrediction( + frame_id=prediction.frame_id, + index=index, + benchmark_class=_benchmark_class(box), + box=box, + ) + for prediction in predictions + for index, box in enumerate(prediction.boxes) + if _center_in_evaluation_range(box.x_m, box.y_m, box.z_m) + ) + per_class: dict[str, dict[str, Any]] = {} + three_d_evaluations: dict[str, _MetricEvaluation] = {} + for class_name in KITTI_BENCHMARK_CLASSES: + bev = _evaluate_metric( + class_name=class_name, + truth_by_frame=truth_by_frame, + predictions=scored, + iou=oriented_bev_iou, + ) + three_d = _evaluate_metric( + class_name=class_name, + truth_by_frame=truth_by_frame, + predictions=scored, + iou=oriented_3d_iou, + ) + three_d_evaluations[class_name] = three_d + per_class[class_name] = { + "iou_threshold": KITTI_IOU_THRESHOLDS[class_name], + "bev_ap40": bev.average_precision_40, + "3d_ap40": three_d.average_precision_40, + "precision": three_d.precision, + "recall": three_d.recall, + "true_positives": three_d.true_positives, + "false_positives": three_d.false_positives, + "ground_truth_count": three_d.ground_truth_count, + } + + all_three_d_pairs = tuple( + pair + for class_name in KITTI_BENCHMARK_CLASSES + for pair in three_d_evaluations[class_name].matched_pairs + ) + errors = _matched_errors(all_three_d_pairs) + total_predictions = sum( + evaluation.true_positives + evaluation.false_positives + for evaluation in three_d_evaluations.values() + ) + total_false_positives = sum( + evaluation.false_positives for evaluation in three_d_evaluations.values() + ) + latencies = np.asarray( + [prediction.inference_ms for prediction in predictions], + dtype=np.float64, + ) + return { + "metric_contract": { + "official_kitti_server_metric": False, + "evaluation_kind": "public-cross-domain-transfer-probe", + "ap_interpolation": "40-point", + "difficulty_filtering": False, + "retuning_on_validation": False, + "model_to_benchmark_class_mapping": MODEL_TO_KITTI_CLASS, + "iou_thresholds": KITTI_IOU_THRESHOLDS, + "model_training_domain": "proprietary-solid-state-lidar", + "model_point_cloud_range": list( + POINTPILLARS_MODEL_POINT_CLOUD_RANGE + ), + "dataset_reference_point_cloud_range": list( + KITTI_REFERENCE_POINT_CLOUD_RANGE + ), + "shared_evaluation_range": list(CROSS_DOMAIN_EVALUATION_RANGE), + "predictions_outside_shared_range_ignored": True, + }, + "frame_count": len(predictions), + "per_class": per_class, + "aggregates": { + "bev_map40": _mean( + [per_class[class_name]["bev_ap40"] for class_name in KITTI_BENCHMARK_CLASSES] + ), + "3d_map40": _mean( + [per_class[class_name]["3d_ap40"] for class_name in KITTI_BENCHMARK_CLASSES] + ), + "false_occupied_rate": ( + total_false_positives / total_predictions + if total_predictions + else 0.0 + ), + "prediction_volume": { + "model_output_box_count": len(all_boxes), + "evaluated_box_count": len(evaluated_boxes), + "outside_shared_range_count": len(all_boxes) + - len(evaluated_boxes), + }, + **errors, + "inference_latency_ms": { + "mean": float(np.mean(latencies)), + "p50": float(np.percentile(latencies, 50)), + "p95": float(np.percentile(latencies, 95)), + "maximum": float(np.max(latencies)), + }, + "distance_bucket_recall": _distance_bucket_recall( + truth_by_frame, + three_d_evaluations, + ), + }, + "claim_boundary": { + "public_cross_domain_transfer_probe": True, + "native_model_accuracy_evaluated": False, + "k1_transfer_evaluated": False, + "navigation_or_safety_accepted": False, + "camera_first_candidate_replaced": False, + }, + } + + +def _read_frame_truth( + labels_zip: zipfile.ZipFile, + calib_zip: zipfile.ZipFile, + frame_id: str, +) -> tuple[KittiLidarTruth, ...]: + calibration = _calibration_transform( + calib_zip.read(f"training/calib/{frame_id}.txt").decode("ascii") + ) + labels = labels_zip.read(f"training/label_2/{frame_id}.txt").decode("ascii") + result: list[KittiLidarTruth] = [] + for line in labels.splitlines(): + fields = line.split() + if not fields or fields[0] not in KITTI_BENCHMARK_CLASSES: + continue + if len(fields) != 15: + raise KittiPointPillarsBenchmarkError("KITTI label row is invalid") + try: + values = np.asarray([float(value) for value in fields[1:]], dtype=np.float64) + except ValueError as exc: + raise KittiPointPillarsBenchmarkError( + "KITTI label row contains invalid numbers" + ) from exc + if not np.isfinite(values).all(): + raise KittiPointPillarsBenchmarkError( + "KITTI label row contains non-finite numbers" + ) + height_m, width_m, length_m = (float(value) for value in values[7:10]) + if height_m <= 0.0 or width_m <= 0.0 or length_m <= 0.0: + raise KittiPointPillarsBenchmarkError("KITTI cuboid dimensions are invalid") + camera_bottom_center = np.asarray( + [values[10], values[11], values[12], 1.0], + dtype=np.float64, + ) + lidar_bottom_center = calibration @ camera_bottom_center + x_m, y_m, z_m = (float(value) for value in lidar_bottom_center[:3]) + z_m += height_m / 2.0 + if not _center_in_evaluation_range(x_m, y_m, z_m): + continue + result.append( + KittiLidarTruth( + frame_id=frame_id, + benchmark_class=fields[0], + x_m=x_m, + y_m=y_m, + z_m=z_m, + length_m=length_m, + width_m=width_m, + height_m=height_m, + yaw_rad=_wrap_angle(-(float(values[13]) + math.pi / 2.0)), + ) + ) + return tuple(result) + + +def _calibration_transform(payload: str) -> np.ndarray: + values: dict[str, np.ndarray] = {} + for line in payload.splitlines(): + if not line.strip(): + continue + key, separator, raw = line.partition(":") + if not separator: + raise KittiPointPillarsBenchmarkError("KITTI calibration row is invalid") + try: + row = np.asarray([float(value) for value in raw.split()], dtype=np.float64) + except ValueError as exc: + raise KittiPointPillarsBenchmarkError( + "KITTI calibration contains invalid numbers" + ) from exc + if not np.isfinite(row).all(): + raise KittiPointPillarsBenchmarkError( + "KITTI calibration contains non-finite numbers" + ) + values[key] = row + if "R0_rect" not in values or values["R0_rect"].size != 9: + raise KittiPointPillarsBenchmarkError("KITTI R0_rect is invalid") + if "Tr_velo_to_cam" not in values or values["Tr_velo_to_cam"].size != 12: + raise KittiPointPillarsBenchmarkError("KITTI Tr_velo_to_cam is invalid") + rectification = np.eye(4, dtype=np.float64) + rectification[:3, :3] = values["R0_rect"].reshape(3, 3) + lidar_to_camera = np.eye(4, dtype=np.float64) + lidar_to_camera[:3, :4] = values["Tr_velo_to_cam"].reshape(3, 4) + try: + return np.linalg.inv(rectification @ lidar_to_camera) + except np.linalg.LinAlgError as exc: + raise KittiPointPillarsBenchmarkError( + "KITTI calibration transform is singular" + ) from exc + + +def _center_in_evaluation_range(x_m: float, y_m: float, z_m: float) -> bool: + return ( + CROSS_DOMAIN_EVALUATION_RANGE[0] + <= x_m + <= CROSS_DOMAIN_EVALUATION_RANGE[3] + and CROSS_DOMAIN_EVALUATION_RANGE[1] + <= y_m + <= CROSS_DOMAIN_EVALUATION_RANGE[4] + and CROSS_DOMAIN_EVALUATION_RANGE[2] + <= z_m + <= CROSS_DOMAIN_EVALUATION_RANGE[5] + ) + + +def _benchmark_class(box: PointPillarsBox) -> str: + try: + return MODEL_TO_KITTI_CLASS[box.model_class] + except KeyError as exc: + raise KittiPointPillarsBenchmarkError( + "PointPillars model class is not admitted" + ) from exc + + +def _truth_as_box(truth: KittiLidarTruth) -> PointPillarsBox: + return PointPillarsBox( + x_m=truth.x_m, + y_m=truth.y_m, + z_m=truth.z_m, + length_m=truth.length_m, + width_m=truth.width_m, + height_m=truth.height_m, + yaw_rad=truth.yaw_rad, + class_id=-1, + model_class=truth.benchmark_class, + score=1.0, + ) + + +def _evaluate_metric( + *, + class_name: str, + truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]], + predictions: tuple[_ScoredPrediction, ...], + iou: Callable[[PointPillarsBox, PointPillarsBox], float], +) -> _MetricEvaluation: + truths = { + frame_id: tuple( + box for box in boxes if box.benchmark_class == class_name + ) + for frame_id, boxes in truth_by_frame.items() + } + ground_truth_count = sum(len(boxes) for boxes in truths.values()) + ordered = sorted( + ( + prediction + for prediction in predictions + if prediction.benchmark_class == class_name + ), + key=lambda prediction: ( + -prediction.box.score, + prediction.frame_id, + prediction.index, + ), + ) + matched: set[tuple[str, int]] = set() + matched_pairs: list[tuple[PointPillarsBox, KittiLidarTruth]] = [] + true_positive_flags: list[int] = [] + false_positive_flags: list[int] = [] + threshold = KITTI_IOU_THRESHOLDS[class_name] + for prediction in ordered: + candidates = truths[prediction.frame_id] + best_index = -1 + best_iou = -1.0 + for truth_index, truth in enumerate(candidates): + if (prediction.frame_id, truth_index) in matched: + continue + overlap = iou(prediction.box, _truth_as_box(truth)) + if overlap > best_iou: + best_iou = overlap + best_index = truth_index + if best_index >= 0 and best_iou >= threshold: + matched.add((prediction.frame_id, best_index)) + matched_pairs.append((prediction.box, candidates[best_index])) + true_positive_flags.append(1) + false_positive_flags.append(0) + else: + true_positive_flags.append(0) + false_positive_flags.append(1) + cumulative_true = np.cumsum(true_positive_flags, dtype=np.float64) + cumulative_false = np.cumsum(false_positive_flags, dtype=np.float64) + precision = np.divide( + cumulative_true, + np.maximum(cumulative_true + cumulative_false, 1.0), + ) + recall = cumulative_true / max(float(ground_truth_count), 1.0) + average_precision = _ap40(precision, recall) + return _MetricEvaluation( + average_precision_40=average_precision, + precision=float(precision[-1]) if precision.size else 0.0, + recall=float(recall[-1]) if recall.size else 0.0, + true_positives=int(cumulative_true[-1]) if cumulative_true.size else 0, + false_positives=int(cumulative_false[-1]) if cumulative_false.size else 0, + ground_truth_count=ground_truth_count, + matched_truth=frozenset(matched), + matched_pairs=tuple(matched_pairs), + ) + + +def _ap40(precision: np.ndarray, recall: np.ndarray) -> float: + if precision.size == 0: + return 0.0 + interpolated = [ + float(np.max(precision[recall >= threshold])) + if np.any(recall >= threshold) + else 0.0 + for threshold in np.arange(40, dtype=np.float64) / 40.0 + ] + return _mean(interpolated) + + +def _matched_errors( + pairs: tuple[tuple[PointPillarsBox, KittiLidarTruth], ...], +) -> dict[str, Any]: + center_errors = [ + math.dist( + (prediction.x_m, prediction.y_m, prediction.z_m), + (truth.x_m, truth.y_m, truth.z_m), + ) + for prediction, truth in pairs + ] + range_errors = [ + abs( + math.hypot(prediction.x_m, prediction.y_m) + - math.hypot(truth.x_m, truth.y_m) + ) + for prediction, truth in pairs + ] + yaw_errors = [ + abs(_wrap_angle(prediction.yaw_rad - truth.yaw_rad)) + for prediction, truth in pairs + ] + return { + "center_error_m": _error_summary(center_errors), + "range_error_m": _error_summary(range_errors), + "yaw_error_rad": _error_summary(yaw_errors), + } + + +def _error_summary(values: list[float]) -> dict[str, float | int | None]: + if not values: + return {"count": 0, "mean": None, "p95": None, "maximum": None} + array = np.asarray(values, dtype=np.float64) + return { + "count": len(values), + "mean": float(np.mean(array)), + "p95": float(np.percentile(array, 95)), + "maximum": float(np.max(array)), + } + + +def _distance_bucket_recall( + truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]], + evaluations: dict[str, _MetricEvaluation], +) -> dict[str, dict[str, float | int]]: + result: dict[str, dict[str, float | int]] = {} + for minimum, maximum in DISTANCE_BUCKETS_M: + total = 0 + matched = 0 + for frame_id, truths in truth_by_frame.items(): + by_class_index = {class_name: 0 for class_name in KITTI_BENCHMARK_CLASSES} + for truth in truths: + index = by_class_index[truth.benchmark_class] + by_class_index[truth.benchmark_class] += 1 + distance = math.hypot(truth.x_m, truth.y_m) + if not minimum <= distance < maximum: + continue + total += 1 + if (frame_id, index) in evaluations[ + truth.benchmark_class + ].matched_truth: + matched += 1 + result[f"{int(minimum)}-{int(maximum)}m"] = { + "ground_truth_count": total, + "matched_count": matched, + "recall": matched / total if total else 0.0, + } + return result + + +def _wrap_angle(value: float) -> float: + return (value + math.pi) % (2.0 * math.pi) - math.pi + + +def _mean(values: list[float]) -> float: + return sum(values) / len(values) if values else 0.0 diff --git a/src/k1link/compute/l3_pointpillars_admission.py b/src/k1link/compute/l3_pointpillars_admission.py new file mode 100644 index 0000000..dcb3974 --- /dev/null +++ b/src/k1link/compute/l3_pointpillars_admission.py @@ -0,0 +1,828 @@ +"""Fail-closed admission for the L3 NVIDIA PointPillars benchmark. + +The admission deliberately separates three different claims: + +* a detector-quality baseline needs independent oriented 3D box truth; +* K1 transfer can measure runtime and representation stability only after that + baseline has been accepted; +* point-wise semantic or instance labels are valuable evidence, but are not a + substitute for 3D cuboid truth. + +The result is immutable and content addressed. It grants no command, +navigation, or safety authority and cannot start a worker or install a model. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +from k1link.artifacts import utc_now_iso +from k1link.compute.pointpillars_postprocess import ( + POINTPILLARS_EMBEDDED_SCORE_THRESHOLD, + POINTPILLARS_MODEL_POINT_CLOUD_RANGE, +) + +L3_PROFILE_SCHEMA: Final = "missioncore.l3-pointpillars-benchmark-profile/v1" +L3_DATASET_INVENTORY_SCHEMA: Final = "missioncore.l3-lidar-dataset-inventory/v1" +L3_WORKER_INVENTORY_SCHEMA: Final = "missioncore.l3-worker-inventory/v1" +L3_ADMISSION_SCHEMA: Final = "missioncore.l3-pointpillars-admission/v1" +L3_REPORT_SCHEMA: Final = "missioncore.l3-pointpillars-admission-report/v1" + +L3_REPORT_NAME: Final = "admission-report.json" +L3_MANIFEST_NAME: Final = "manifest.json" + +_RESULT_ID = re.compile(r"^l3-pointpillars-admission-[a-f0-9]{64}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,191}$") + +_BOX_ANNOTATION = "oriented-3d-boxes" +_ONNX_CONTRACT_SHA256 = ( + "2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be" +) +_POINT_FIELDS = ("x", "y", "z", "intensity") +_MODEL_CLASSES = ("Vehicle", "Pedestrian", "Cyclist") +_BENCHMARK_CLASSES = ("Car", "Pedestrian", "Cyclist") +_BENCHMARK_CLASS_MAPPING = { + "Vehicle": "Car", + "Pedestrian": "Pedestrian", + "Cyclist": "Cyclist", +} +_POSTPROCESSING_CONTRACT = { + "reference_repository": "https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes", + "reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8", + "output_row_fields": [ + "x", + "y", + "z", + "length", + "width", + "height", + "yaw", + "class_id", + "score", + ], + "class_agnostic_nms": True, + "nms_iou_threshold": 0.01, + "pre_nms_top_n": 4096, + "embedded_score_threshold": POINTPILLARS_EMBEDDED_SCORE_THRESHOLD, + "embedded_contract_source": "onnx-node-attributes", +} +_BENCHMARK_METRIC_CONTRACT = { + "official_kitti_server_metric": False, + "evaluation_kind": "public-cross-domain-transfer-probe", + "ap_interpolation": "40-point", + "difficulty_filtering": False, + "predictions_outside_shared_range_ignored": True, + "iou_thresholds": { + "Car": 0.7, + "Pedestrian": 0.5, + "Cyclist": 0.5, + }, + "distance_buckets_m": [[0, 20], [20, 40], [40, 70]], +} +_POINT_CLOUD_RANGE = POINTPILLARS_MODEL_POINT_CLOUD_RANGE +_PUBLIC_TRANSFER_METRICS = ( + "bev-map", + "3d-map", + "center-error-m", + "range-error-m", + "yaw-error-rad", + "distance-bucket-recall", + "false-occupied-rate", + "end-to-end-latency-ms", +) +_TRANSFER_METRICS = ( + "input-admission-rate", + "output-schema-valid-rate", + "deterministic-replay-rate", + "end-to-end-latency-ms", + "queue-wait-ms", + "drop-rate", +) + + +class L3PointPillarsAdmissionError(RuntimeError): + """An L3 profile, inventory, or immutable result is invalid.""" + + +@dataclass(frozen=True, slots=True) +class L3PointPillarsAdmission: + result_root: Path + result_id: str + manifest: dict[str, Any] + report: dict[str, Any] + + @property + def public_transfer_probe_authorized(self) -> bool: + decision = _object(self.report.get("decision"), "L3 decision") + return decision.get("public_cross_domain_probe_authorized") is True + + +def build_l3_pointpillars_admission( + *, + profile_path: Path, + dataset_inventory_path: Path, + worker_inventory_path: Path, + output_root: Path, +) -> L3PointPillarsAdmission: + """Build or reopen one immutable L3 benchmark admission.""" + + profile_path = profile_path.resolve(strict=True) + dataset_inventory_path = dataset_inventory_path.resolve(strict=True) + worker_inventory_path = worker_inventory_path.resolve(strict=True) + profile = _read_json(profile_path) + dataset_inventory = _read_json(dataset_inventory_path) + worker_inventory = _read_json(worker_inventory_path) + _validate_profile(profile) + _validate_dataset_inventory(dataset_inventory) + _validate_worker_inventory(worker_inventory) + + identity = { + "schema_version": L3_ADMISSION_SCHEMA, + "profile": profile, + "profile_sha256": _sha256(profile_path), + "dataset_inventory_sha256": _sha256(dataset_inventory_path), + "worker_inventory_sha256": _sha256(worker_inventory_path), + "producer_sha256": _sha256(Path(__file__)), + "authority": _authority(), + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + result_id = f"l3-pointpillars-admission-{identity_sha256}" + destination = output_root.expanduser().absolute() + destination.mkdir(mode=0o700, parents=True, exist_ok=True) + result_root = destination / result_id + if result_root.exists(): + return read_l3_pointpillars_admission(result_root) + + report = _build_report( + result_id=result_id, + profile=profile, + dataset_inventory=dataset_inventory, + worker_inventory=worker_inventory, + ) + staging = destination / f".{result_id}.{os.getpid()}.incomplete" + staging.mkdir(mode=0o700, exist_ok=False) + try: + _write_json(staging / L3_REPORT_NAME, report) + report_artifact = _artifact( + staging / L3_REPORT_NAME, + "l3-pointpillars-admission-report", + ) + manifest = { + "schema_version": L3_ADMISSION_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "status": report["status"], + "artifacts": [report_artifact], + "created_at_utc": utc_now_iso(), + "authority": _authority(), + } + _write_json(staging / L3_MANIFEST_NAME, manifest) + os.replace(staging, result_root) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return read_l3_pointpillars_admission(result_root) + + +def read_l3_pointpillars_admission(root: Path) -> L3PointPillarsAdmission: + """Read and fully validate an immutable L3 admission result.""" + + resolved = root.expanduser().resolve(strict=True) + if not resolved.is_dir() or _RESULT_ID.fullmatch(resolved.name) is None: + raise L3PointPillarsAdmissionError("L3 result root is invalid") + manifest = _read_json(resolved / L3_MANIFEST_NAME) + report = _read_json(resolved / L3_REPORT_NAME) + if ( + manifest.get("schema_version") != L3_ADMISSION_SCHEMA + or manifest.get("result_id") != resolved.name + or report.get("schema_version") != L3_REPORT_SCHEMA + or report.get("result_id") != resolved.name + or manifest.get("status") != report.get("status") + or manifest.get("authority") != _authority() + or report.get("authority") != _authority() + ): + raise L3PointPillarsAdmissionError("L3 manifest and report are inconsistent") + identity = _object(manifest.get("identity"), "L3 identity") + expected_identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + if ( + manifest.get("identity_sha256") != expected_identity_sha256 + or resolved.name != f"l3-pointpillars-admission-{expected_identity_sha256}" + ): + raise L3PointPillarsAdmissionError("L3 result identity is invalid") + artifacts = _array(manifest.get("artifacts"), "L3 artifacts") + if len(artifacts) != 1: + raise L3PointPillarsAdmissionError("L3 result has an invalid artifact set") + artifact = _object(artifacts[0], "L3 report artifact") + report_path = resolved / L3_REPORT_NAME + if ( + artifact.get("role") != "l3-pointpillars-admission-report" + or artifact.get("path") != L3_REPORT_NAME + or artifact.get("media_type") != "application/json" + or artifact.get("sha256") != _sha256(report_path) + or artifact.get("byte_length") != report_path.stat().st_size + ): + raise L3PointPillarsAdmissionError("L3 report artifact is invalid") + _validate_report(report) + return L3PointPillarsAdmission( + result_root=resolved, + result_id=resolved.name, + manifest=manifest, + report=report, + ) + + +def _build_report( + *, + result_id: str, + profile: dict[str, Any], + dataset_inventory: dict[str, Any], + worker_inventory: dict[str, Any], +) -> dict[str, Any]: + detector = _object(profile.get("detector"), "L3 detector") + runtime_policy = _object(profile.get("runtime_policy"), "L3 runtime policy") + public_probe = _object( + profile.get("public_cross_domain_probe"), + "L3 public cross-domain probe", + ) + required_model = _required_string(detector, "triton_model_name") + candidate_frozen = detector.get("candidate_frozen") is True + required_version = detector.get("candidate_model_version") + required_source_sha256 = detector.get("candidate_source_sha256") + required_label_sha256 = detector.get("candidate_label_sha256") + + triton = _object(worker_inventory.get("triton"), "L3 Triton inventory") + installed_models = [ + _object(item, "L3 worker model") + for item in _array(triton.get("models"), "L3 worker models") + ] + matching_models = [ + model + for model in installed_models + if candidate_frozen + and model.get("name") == required_model + and model.get("upstream_version") == required_version + and model.get("source_model_sha256") == required_source_sha256 + and model.get("source_label_sha256") == required_label_sha256 + ] + model_ready = any(_model_ready(model) for model in matching_models) + staged_models = [ + _object(item, "L3 staged worker model") + for item in _array( + worker_inventory.get("staged_models"), + "L3 staged worker models", + ) + ] + matching_staged_models = [ + model + for model in staged_models + if candidate_frozen + and model.get("name") == required_model + and model.get("upstream_version") == required_version + and model.get("source_model_sha256") == required_source_sha256 + and model.get("source_label_sha256") == required_label_sha256 + ] + staged_model_ready = any(_model_ready(model) for model in matching_staged_models) + + datasets = [ + _object(item, "L3 dataset") + for item in _array(dataset_inventory.get("datasets"), "L3 datasets") + ] + dataset_findings = [_dataset_finding(dataset) for dataset in datasets] + eligible_datasets = [ + dataset + for dataset in datasets + if _dataset_supports_public_probe(dataset, public_probe) + ] + + runtime_checks = { + "canonical_triton_healthy": triton.get("healthy") is True, + "canonical_triton_image_pinned": ( + triton.get("image") == runtime_policy.get("triton_image") + and triton.get("image_digest") == runtime_policy.get("triton_image_digest") + ), + "explicit_model_control": triton.get("model_control_mode") == "explicit", + "strict_readiness": triton.get("strict_readiness") is True, + "model_repository_read_only": triton.get("model_repository_read_only") is True, + "second_serving_stack_absent": ( + worker_inventory.get("serving_stack_count") == 1 + ), + } + blockers: list[str] = [] + if not all(runtime_checks.values()): + blockers.append("canonical-triton-runtime-policy-not-satisfied") + if not eligible_datasets: + blockers.append("public-oriented-3d-box-truth-not-admitted") + if not candidate_frozen: + blockers.append("pointpillars-compatible-candidate-not-frozen") + elif not matching_models: + blockers.append( + "pointpillars-model-not-installed-live" + if staged_model_ready + else "pointpillars-model-artifact-not-installed" + ) + elif not model_ready: + blockers.append("pointpillars-target-engine-or-provenance-not-verified") + + public_probe_authorized = not blockers + status = ( + "ready-for-public-cross-domain-probe" + if public_probe_authorized + else "blocked-foundation-assets" + ) + next_gate = ( + "run-public-cross-domain-pointpillars-probe" + if public_probe_authorized + else _next_gate(blockers) + ) + return { + "schema_version": L3_REPORT_SCHEMA, + "result_id": result_id, + "status": status, + "profile_id": profile["profile_id"], + "observations": { + "dataset_inventory_observed_at_utc": dataset_inventory["observed_at_utc"], + "worker_inventory_observed_at_utc": worker_inventory["observed_at_utc"], + "worker_host_id": worker_inventory["host_id"], + }, + "detector": { + "family": detector["family"], + "upstream_model_id": detector["upstream_model_id"], + "candidate_frozen": candidate_frozen, + "candidate_model_version": required_version, + "candidate_source_sha256": required_source_sha256, + "candidate_label_sha256": required_label_sha256, + "triton_model_name": required_model, + "required_input_fields": list(_POINT_FIELDS), + "maximum_points": detector["maximum_points"], + "point_cloud_range": list(_POINT_CLOUD_RANGE), + "training_domain": detector["training_domain"], + "training_ground_truth_publicly_reproducible": False, + "model_classes": list(_MODEL_CLASSES), + "benchmark_classes": list(_BENCHMARK_CLASSES), + "benchmark_class_mapping": _BENCHMARK_CLASS_MAPPING, + "benchmark_metric_contract": _BENCHMARK_METRIC_CONTRACT, + "postprocessing": _POSTPROCESSING_CONTRACT, + "installed_matching_model_count": len(matching_models), + "model_ready": model_ready, + "staged_matching_model_count": len(matching_staged_models), + "staged_target_engine_ready": staged_model_ready, + }, + "runtime_checks": runtime_checks, + "dataset_findings": dataset_findings, + "eligible_public_probe_dataset_ids": [ + _required_string(dataset, "dataset_id") for dataset in eligible_datasets + ], + "blocker_codes": blockers, + "decision": { + "public_cross_domain_probe_authorized": public_probe_authorized, + "native_model_accuracy_claim_authorized": False, + "k1_transfer_stability_authorized": False, + "k1_transfer_quality_claim_authorized": False, + "semantic_point_labels_substitute_for_3d_boxes": False, + "fine_tuning_allowed": False, + "second_serving_stack_allowed": False, + "lab_publication_allowed": False, + "centerpoint_comparison_allowed": False, + }, + "claim_boundaries": { + "public_cross_domain_probe_metrics": list(_PUBLIC_TRANSFER_METRICS), + "k1_transfer_stability_metrics": list(_TRANSFER_METRICS), + "public_probe_is_native_model_accuracy": False, + "k1_accuracy_requires_independent_truth": True, + "absence_of_detection_means_free_space": False, + "point_instance_clusters_are_3d_box_truth": False, + }, + "next_gate": next_gate, + "authority": _authority(), + } + + +def _dataset_finding(dataset: dict[str, Any]) -> dict[str, Any]: + annotations = set(_string_array(dataset.get("annotations"), "dataset annotations")) + installed = dataset.get("installed") is True + has_box_truth = _BOX_ANNOTATION in annotations + return { + "dataset_id": _required_string(dataset, "dataset_id"), + "installed": installed, + "point_fields": _string_array(dataset.get("point_fields"), "dataset point fields"), + "annotations": sorted(annotations), + "independent_ground_truth": dataset.get("independent_ground_truth") is True, + "oriented_3d_box_accuracy_eligible": ( + installed + and has_box_truth + and dataset.get("independent_ground_truth") is True + and set(_POINT_FIELDS).issubset( + set(_string_array(dataset.get("point_fields"), "dataset point fields")) + ) + ), + "semantic_or_instance_labels_are_not_boxes": ( + not has_box_truth + and bool( + annotations.intersection( + {"point-semantic-labels", "point-instance-labels"} + ) + ) + ), + } + + +def _dataset_supports_public_probe( + dataset: dict[str, Any], + public_probe: dict[str, Any], +) -> bool: + splits = set(_string_array(dataset.get("splits"), "dataset splits")) + point_fields = set( + _string_array(dataset.get("point_fields"), "dataset point fields") + ) + annotations = set( + _string_array(dataset.get("annotations"), "dataset annotations") + ) + required_split = _required_string(public_probe, "required_split") + return ( + dataset.get("installed") is True + and dataset.get("independent_ground_truth") is True + and required_split in splits + and set(_POINT_FIELDS).issubset(point_fields) + and _BOX_ANNOTATION in annotations + ) + + +def _model_ready(model: dict[str, Any]) -> bool: + smoke = model.get("representation_smoke") + return ( + _valid_sha256(model.get("artifact_sha256")) + and model.get("backend") == "tensorrt" + and model.get("precision") == "strongly-typed" + and model.get("engine_built_on_target") is True + and model.get("provenance_verified") is True + and model.get("source_format") == "onnx" + and model.get("input_fields") == list(_POINT_FIELDS) + and model.get("maximum_points") == 204_800 + and model.get("point_cloud_range") == list(_POINT_CLOUD_RANGE) + and model.get("model_classes") == list(_MODEL_CLASSES) + and _valid_sha256(model.get("source_label_sha256")) + and model.get("outputs") + == [ + { + "name": "output_boxes", + "dtype": "FP32", + "shape": [1, 393_216, 9], + }, + {"name": "num_boxes", "dtype": "INT32", "shape": [1]}, + ] + and isinstance(smoke, dict) + and smoke.get("status") == "engine-executed" + and _valid_sha256(smoke.get("input_artifact_sha256")) + and smoke.get("input_point_count") == 169_883 + and isinstance(smoke.get("single_query_gpu_compute_ms"), float) + and 0.0 < smoke["single_query_gpu_compute_ms"] < 1000.0 + and smoke.get("accuracy_evaluated") is False + and smoke.get("navigation_or_safety_accepted") is False + ) + + +def _next_gate(blockers: list[str]) -> str: + missing_dataset = "public-oriented-3d-box-truth-not-admitted" in blockers + missing_candidate = "pointpillars-compatible-candidate-not-frozen" in blockers + missing_model = "pointpillars-model-artifact-not-installed" in blockers + staged_model = "pointpillars-model-not-installed-live" in blockers + if missing_dataset and missing_candidate: + return "admit-public-3d-box-split-and-freeze-compatible-pointpillars-candidate" + if missing_dataset and missing_model: + return "admit-public-3d-box-split-and-build-target-pointpillars-engine" + if missing_dataset and staged_model: + return "admit-public-3d-box-split-then-install-staged-pointpillars-model" + if missing_dataset: + return "admit-public-oriented-3d-box-validation-split" + if missing_model: + return "build-and-admit-target-pointpillars-engine" + if staged_model: + return "install-staged-pointpillars-model-in-canonical-triton" + if missing_candidate: + return "freeze-compatible-pointpillars-onnx-candidate" + if "pointpillars-target-engine-or-provenance-not-verified" in blockers: + return "verify-target-engine-and-model-provenance" + return "repair-canonical-triton-runtime-policy" + + +def _validate_profile(profile: dict[str, Any]) -> None: + if profile.get("schema_version") != L3_PROFILE_SCHEMA: + raise L3PointPillarsAdmissionError("L3 profile schema is invalid") + _safe_identifier(_required_string(profile, "profile_id"), "L3 profile id") + detector = _object(profile.get("detector"), "L3 detector") + if ( + detector.get("family") != "nvidia-tao-pointpillars" + or detector.get("upstream_model_id") != "nvidia/tao/pointpillarnet" + or detector.get("triton_model_name") != "pointpillars" + or detector.get("required_source_format") != "onnx" + or detector.get("input_representation") != "native-sensor-scan" + or detector.get("input_coordinate_frame") != "sensor/lidar" + or detector.get("input_fields") != list(_POINT_FIELDS) + or detector.get("batch_size") != 1 + or detector.get("maximum_points") != 204_800 + or detector.get("point_cloud_range") != list(_POINT_CLOUD_RANGE) + or detector.get("training_domain") != "proprietary-solid-state-lidar" + or detector.get("training_ground_truth_publicly_reproducible") is not False + or detector.get("onnx_contract_sha256") != _ONNX_CONTRACT_SHA256 + or detector.get("model_classes") != list(_MODEL_CLASSES) + ): + raise L3PointPillarsAdmissionError("L3 detector contract is invalid") + candidate_frozen = detector.get("candidate_frozen") + candidate_version = detector.get("candidate_model_version") + candidate_sha256 = detector.get("candidate_source_sha256") + candidate_label_sha256 = detector.get("candidate_label_sha256") + if ( + not isinstance(candidate_frozen, bool) + or ( + candidate_frozen + and ( + not isinstance(candidate_version, str) + or not candidate_version + or not _valid_sha256(candidate_sha256) + or not _valid_sha256(candidate_label_sha256) + ) + ) + or ( + not candidate_frozen + and ( + candidate_version is not None + or candidate_sha256 is not None + or candidate_label_sha256 is not None + ) + ) + ): + raise L3PointPillarsAdmissionError("L3 detector candidate freeze is invalid") + if detector.get("postprocessing") != _POSTPROCESSING_CONTRACT: + raise L3PointPillarsAdmissionError("L3 detector postprocessing is invalid") + runtime = _object(profile.get("runtime_policy"), "L3 runtime policy") + if ( + runtime.get("existing_triton_only") is not True + or runtime.get("second_serving_stack_allowed") is not False + or runtime.get("engine_built_on_target_required") is not True + or runtime.get("precision") != "strongly-typed" + or runtime.get("triton_image") != "nvcr.io/nvidia/tritonserver:26.06-py3" + or not _valid_sha256(runtime.get("triton_image_digest")) + ): + raise L3PointPillarsAdmissionError("L3 runtime policy is invalid") + public_probe = _object( + profile.get("public_cross_domain_probe"), + "L3 public cross-domain probe", + ) + if ( + public_probe.get("required_split") != "validation" + or public_probe.get("required_ground_truth") != _BOX_ANNOTATION + or public_probe.get("independent_ground_truth_required") is not True + or public_probe.get("benchmark_classes") != list(_BENCHMARK_CLASSES) + or public_probe.get("model_to_benchmark_class_mapping") + != _BENCHMARK_CLASS_MAPPING + or public_probe.get("metric_contract") != _BENCHMARK_METRIC_CONTRACT + or public_probe.get("metrics") != list(_PUBLIC_TRANSFER_METRICS) + or public_probe.get("retuning_allowed") is not False + or public_probe.get("native_accuracy_claim_allowed") is not False + ): + raise L3PointPillarsAdmissionError( + "L3 public cross-domain probe is invalid" + ) + transfer = _object(profile.get("k1_transfer_stability"), "L3 transfer gate") + if ( + transfer.get("requires_completed_public_cross_domain_probe") is not True + or transfer.get("metrics") != list(_TRANSFER_METRICS) + or transfer.get("accuracy_claim_allowed") is not False + or transfer.get("retuning_allowed") is not False + ): + raise L3PointPillarsAdmissionError("L3 K1 transfer gate is invalid") + if profile.get("authority") != _authority(): + raise L3PointPillarsAdmissionError("L3 profile authority is invalid") + + +def _validate_dataset_inventory(inventory: dict[str, Any]) -> None: + if inventory.get("schema_version") != L3_DATASET_INVENTORY_SCHEMA: + raise L3PointPillarsAdmissionError("L3 dataset inventory schema is invalid") + _utc(_required_string(inventory, "observed_at_utc"), "dataset observation time") + datasets = _array(inventory.get("datasets"), "L3 datasets") + if not datasets: + raise L3PointPillarsAdmissionError("L3 dataset inventory is empty") + seen: set[str] = set() + for raw_dataset in datasets: + dataset = _object(raw_dataset, "L3 dataset") + dataset_id = _safe_identifier( + _required_string(dataset, "dataset_id"), + "L3 dataset id", + ) + if dataset_id in seen: + raise L3PointPillarsAdmissionError("L3 dataset inventory has duplicates") + seen.add(dataset_id) + installed = dataset.get("installed") + if not isinstance(installed, bool): + raise L3PointPillarsAdmissionError("L3 dataset installed flag is invalid") + release_identity = dataset.get("release_identity_sha256") + if installed and not _valid_sha256(release_identity): + raise L3PointPillarsAdmissionError( + "installed L3 dataset lacks a release identity" + ) + if not installed and release_identity is not None: + raise L3PointPillarsAdmissionError( + "absent L3 dataset cannot claim a release identity" + ) + point_fields = _string_array( + dataset.get("point_fields"), + "dataset point fields", + ) + if any(field not in _POINT_FIELDS for field in point_fields): + raise L3PointPillarsAdmissionError("L3 dataset point field is unknown") + _string_array(dataset.get("annotations"), "dataset annotations") + _string_array(dataset.get("splits"), "dataset splits") + if not isinstance(dataset.get("independent_ground_truth"), bool): + raise L3PointPillarsAdmissionError( + "L3 dataset truth independence flag is invalid" + ) + _required_string(dataset, "license") + + +def _validate_worker_inventory(inventory: dict[str, Any]) -> None: + if inventory.get("schema_version") != L3_WORKER_INVENTORY_SCHEMA: + raise L3PointPillarsAdmissionError("L3 worker inventory schema is invalid") + _safe_identifier(_required_string(inventory, "host_id"), "L3 worker host id") + _utc(_required_string(inventory, "observed_at_utc"), "worker observation time") + serving_stack_count = inventory.get("serving_stack_count") + if ( + isinstance(serving_stack_count, bool) + or not isinstance(serving_stack_count, int) + or serving_stack_count < 0 + ): + raise L3PointPillarsAdmissionError("L3 serving stack count is invalid") + triton = _object(inventory.get("triton"), "L3 Triton inventory") + _required_string(triton, "container_name") + _required_string(triton, "image") + if not _valid_sha256(triton.get("image_digest")): + raise L3PointPillarsAdmissionError("L3 Triton image digest is invalid") + for key in ("healthy", "strict_readiness", "model_repository_read_only"): + if not isinstance(triton.get(key), bool): + raise L3PointPillarsAdmissionError(f"L3 Triton {key} flag is invalid") + _required_string(triton, "model_control_mode") + models = _array(triton.get("models"), "L3 worker models") + seen: set[str] = set() + for raw_model in models: + model = _object(raw_model, "L3 worker model") + name = _safe_identifier(_required_string(model, "name"), "L3 worker model name") + if name in seen: + raise L3PointPillarsAdmissionError("L3 worker inventory has duplicate models") + seen.add(name) + if not _valid_sha256(model.get("artifact_sha256")): + raise L3PointPillarsAdmissionError("L3 worker model identity is invalid") + _required_string(model, "backend") + staged_models = _array(inventory.get("staged_models"), "L3 staged worker models") + staged_seen: set[str] = set() + for raw_model in staged_models: + model = _object(raw_model, "L3 staged worker model") + name = _safe_identifier( + _required_string(model, "name"), + "L3 staged worker model name", + ) + if name in staged_seen: + raise L3PointPillarsAdmissionError( + "L3 worker inventory has duplicate staged models" + ) + staged_seen.add(name) + if not _valid_sha256(model.get("artifact_sha256")): + raise L3PointPillarsAdmissionError( + "L3 staged worker model identity is invalid" + ) + _required_string(model, "backend") + gpu = _object(inventory.get("gpu"), "L3 GPU inventory") + _required_string(gpu, "name") + _required_string(gpu, "driver_version") + memory_mib = gpu.get("memory_total_mib") + if ( + isinstance(memory_mib, bool) + or not isinstance(memory_mib, int) + or memory_mib <= 0 + ): + raise L3PointPillarsAdmissionError("L3 GPU memory declaration is invalid") + + +def _validate_report(report: dict[str, Any]) -> None: + if report.get("schema_version") != L3_REPORT_SCHEMA: + raise L3PointPillarsAdmissionError("L3 report schema is invalid") + decision = _object(report.get("decision"), "L3 decision") + public_probe = decision.get("public_cross_domain_probe_authorized") + transfer = decision.get("k1_transfer_stability_authorized") + blockers = _string_array(report.get("blocker_codes"), "L3 blockers") + if ( + not isinstance(public_probe, bool) + or not isinstance(transfer, bool) + or transfer is not False + or decision.get("native_model_accuracy_claim_authorized") is not False + or decision.get("k1_transfer_quality_claim_authorized") is not False + or decision.get("semantic_point_labels_substitute_for_3d_boxes") is not False + or decision.get("fine_tuning_allowed") is not False + or decision.get("second_serving_stack_allowed") is not False + or decision.get("lab_publication_allowed") is not False + or decision.get("centerpoint_comparison_allowed") is not False + or (public_probe and blockers) + or (not public_probe and not blockers) + or report.get("authority") != _authority() + ): + raise L3PointPillarsAdmissionError("L3 report decision is invalid") + + +def _authority() -> dict[str, bool]: + return { + "shadow_only": True, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + } + + +def _artifact(path: Path, role: str) -> dict[str, object]: + return { + "role": role, + "path": path.name, + "media_type": "application/json", + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + + +def _read_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise L3PointPillarsAdmissionError( + f"cannot read L3 JSON: {path.name}" + ) from exc + return _object(value, f"L3 JSON {path.name}") + + +def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _valid_sha256(value: object) -> bool: + return isinstance(value, str) and _SHA256.fullmatch(value) is not None + + +def _safe_identifier(value: str, label: str) -> str: + if _IDENTIFIER.fullmatch(value) is None: + raise L3PointPillarsAdmissionError(f"{label} is invalid") + return value + + +def _utc(value: str, label: str) -> None: + if not value.endswith("Z") or "T" not in value: + raise L3PointPillarsAdmissionError(f"{label} is invalid") + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise L3PointPillarsAdmissionError(f"{label} must be an object") + return value + + +def _array(value: object, label: str) -> list[Any]: + if not isinstance(value, list): + raise L3PointPillarsAdmissionError(f"{label} must be an array") + return value + + +def _string_array(value: object, label: str) -> list[str]: + values = _array(value, label) + if any(not isinstance(item, str) or not item for item in values): + raise L3PointPillarsAdmissionError(f"{label} must contain strings") + return values + + +def _required_string(value: dict[str, Any], key: str) -> str: + item = value.get(key) + if not isinstance(item, str) or not item.strip(): + raise L3PointPillarsAdmissionError(f"L3 {key} is invalid") + return item diff --git a/src/k1link/compute/pointpillars_postprocess.py b/src/k1link/compute/pointpillars_postprocess.py new file mode 100644 index 0000000..296a693 --- /dev/null +++ b/src/k1link/compute/pointpillars_postprocess.py @@ -0,0 +1,321 @@ +"""Validated NVIDIA PointPillars candidate decoding and sample-compatible NMS.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Final + +import numpy as np + +POINTPILLARS_MODEL_CLASSES: Final = ("Vehicle", "Pedestrian", "Cyclist") +POINTPILLARS_OUTPUT_FIELDS: Final = ( + "x", + "y", + "z", + "length", + "width", + "height", + "yaw", + "class_id", + "score", +) +POINTPILLARS_NMS_IOU_THRESHOLD: Final = 0.01 +POINTPILLARS_PRE_NMS_TOP_N: Final = 4_096 +POINTPILLARS_EMBEDDED_SCORE_THRESHOLD: Final = 0.1 +POINTPILLARS_MODEL_POINT_CLOUD_RANGE: Final = ( + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432, +) +NVIDIA_REFERENCE_COMMIT: Final = "a540badc47812a17a94e924b537d49ad3969b5a8" +_EPSILON: Final = 1e-8 + + +class PointPillarsPostprocessError(RuntimeError): + """PointPillars output does not satisfy the frozen NVIDIA contract.""" + + +@dataclass(frozen=True, slots=True) +class PointPillarsBox: + x_m: float + y_m: float + z_m: float + length_m: float + width_m: float + height_m: float + yaw_rad: float + class_id: int + model_class: str + score: float + + +@dataclass(frozen=True, slots=True) +class _Geometry: + corners: tuple[tuple[float, float], ...] + area: float + minimum_x: float + maximum_x: float + minimum_y: float + maximum_y: float + + +def decode_pointpillars_output( + output_boxes: np.ndarray, + num_boxes: np.ndarray, + *, + pre_nms_top_n: int = POINTPILLARS_PRE_NMS_TOP_N, + nms_iou_threshold: float = POINTPILLARS_NMS_IOU_THRESHOLD, +) -> tuple[PointPillarsBox, ...]: + """Decode validated rows and reproduce the NVIDIA sample's class-agnostic NMS.""" + + boxes = np.asarray(output_boxes) + counts = np.asarray(num_boxes) + if boxes.shape != (1, 393_216, 9) or boxes.dtype != np.float32: + raise PointPillarsPostprocessError("PointPillars output_boxes contract changed") + if counts.shape != (1,) or counts.dtype != np.int32: + raise PointPillarsPostprocessError("PointPillars num_boxes contract changed") + count = int(counts[0]) + if count < 0 or count > boxes.shape[1]: + raise PointPillarsPostprocessError("PointPillars candidate count is invalid") + if ( + isinstance(pre_nms_top_n, bool) + or not isinstance(pre_nms_top_n, int) + or pre_nms_top_n < 1 + or not math.isfinite(nms_iou_threshold) + or not 0.0 <= nms_iou_threshold <= 1.0 + ): + raise PointPillarsPostprocessError("PointPillars NMS parameters are invalid") + if count == 0: + return () + + candidates = boxes[0, :count] + if not np.isfinite(candidates).all(): + raise PointPillarsPostprocessError("PointPillars candidate is non-finite") + decoded = tuple(_decode_row(row) for row in candidates) + ordered = tuple( + sorted( + decoded, + key=lambda box: box.score, + reverse=True, + )[:pre_nms_top_n] + ) + return _class_agnostic_nms(ordered, nms_iou_threshold) + + +def oriented_bev_iou(one: PointPillarsBox, another: PointPillarsBox) -> float: + """Return BEV IoU for two validated oriented boxes.""" + + one_geometry = _geometry(one) + another_geometry = _geometry(another) + overlap = _intersection_area(one_geometry, another_geometry) + denominator = one_geometry.area + another_geometry.area - overlap + return overlap / max(denominator, _EPSILON) + + +def oriented_3d_iou(one: PointPillarsBox, another: PointPillarsBox) -> float: + """Return oriented 3D IoU for two center-based LiDAR-frame boxes.""" + + one_geometry = _geometry(one) + another_geometry = _geometry(another) + bev_overlap = _intersection_area(one_geometry, another_geometry) + one_minimum_z = one.z_m - one.height_m / 2.0 + one_maximum_z = one.z_m + one.height_m / 2.0 + another_minimum_z = another.z_m - another.height_m / 2.0 + another_maximum_z = another.z_m + another.height_m / 2.0 + height_overlap = max( + 0.0, + min(one_maximum_z, another_maximum_z) + - max(one_minimum_z, another_minimum_z), + ) + intersection = bev_overlap * height_overlap + one_volume = one_geometry.area * one.height_m + another_volume = another_geometry.area * another.height_m + return intersection / max(one_volume + another_volume - intersection, _EPSILON) + + +def _decode_row(row: np.ndarray) -> PointPillarsBox: + class_value = float(row[7]) + class_id = int(round(class_value)) + if ( + abs(class_value - class_id) > 1e-5 + or class_id < 0 + or class_id >= len(POINTPILLARS_MODEL_CLASSES) + ): + raise PointPillarsPostprocessError("PointPillars class id is invalid") + length_m, width_m, height_m = (float(row[index]) for index in (3, 4, 5)) + score = float(row[8]) + if ( + length_m <= 0.0 + or width_m <= 0.0 + or height_m <= 0.0 + or score < POINTPILLARS_EMBEDDED_SCORE_THRESHOLD + or score > 1.0 + ): + raise PointPillarsPostprocessError( + "PointPillars dimensions or score are invalid" + ) + x_m, y_m, z_m = (float(row[index]) for index in (0, 1, 2)) + return PointPillarsBox( + x_m=x_m, + y_m=y_m, + z_m=z_m, + length_m=length_m, + width_m=width_m, + height_m=height_m, + yaw_rad=float(row[6]), + class_id=class_id, + model_class=POINTPILLARS_MODEL_CLASSES[class_id], + score=score, + ) + + +def _class_agnostic_nms( + boxes: tuple[PointPillarsBox, ...], + threshold: float, +) -> tuple[PointPillarsBox, ...]: + geometries = tuple(_geometry(box) for box in boxes) + suppressed = [False] * len(boxes) + accepted: list[PointPillarsBox] = [] + for index, box in enumerate(boxes): + if suppressed[index]: + continue + accepted.append(box) + geometry = geometries[index] + for candidate_index in range(index + 1, len(boxes)): + if suppressed[candidate_index]: + continue + another = geometries[candidate_index] + if not _aabbs_overlap(geometry, another): + continue + overlap = _intersection_area(geometry, another) + iou = overlap / max(geometry.area + another.area - overlap, _EPSILON) + if iou >= threshold: + suppressed[candidate_index] = True + return tuple(accepted) + + +def _geometry(box: PointPillarsBox) -> _Geometry: + half_length = box.length_m / 2.0 + half_width = box.width_m / 2.0 + cosine = math.cos(box.yaw_rad) + sine = math.sin(box.yaw_rad) + corners = tuple( + ( + box.x_m + local_x * cosine - local_y * sine, + box.y_m + local_x * sine + local_y * cosine, + ) + for local_x, local_y in ( + (-half_length, -half_width), + (half_length, -half_width), + (half_length, half_width), + (-half_length, half_width), + ) + ) + x_values = [point[0] for point in corners] + y_values = [point[1] for point in corners] + return _Geometry( + corners=corners, + area=box.length_m * box.width_m, + minimum_x=min(x_values), + maximum_x=max(x_values), + minimum_y=min(y_values), + maximum_y=max(y_values), + ) + + +def _aabbs_overlap(one: _Geometry, another: _Geometry) -> bool: + return not ( + one.maximum_x < another.minimum_x + or another.maximum_x < one.minimum_x + or one.maximum_y < another.minimum_y + or another.maximum_y < one.minimum_y + ) + + +def _intersection_area(one: _Geometry, another: _Geometry) -> float: + if not _aabbs_overlap(one, another): + return 0.0 + polygon = list(one.corners) + clip = another.corners + for index, edge_start in enumerate(clip): + edge_end = clip[(index + 1) % len(clip)] + polygon = _clip_polygon(polygon, edge_start, edge_end) + if not polygon: + return 0.0 + return abs( + sum( + one_point[0] * another_point[1] + - another_point[0] * one_point[1] + for one_point, another_point in zip( + polygon, + (*polygon[1:], polygon[0]), + strict=True, + ) + ) + ) / 2.0 + + +def _clip_polygon( + polygon: list[tuple[float, float]], + edge_start: tuple[float, float], + edge_end: tuple[float, float], +) -> list[tuple[float, float]]: + if not polygon: + return [] + result: list[tuple[float, float]] = [] + previous = polygon[-1] + previous_inside = _inside(previous, edge_start, edge_end) + for current in polygon: + current_inside = _inside(current, edge_start, edge_end) + if current_inside: + if not previous_inside: + result.append(_line_intersection(previous, current, edge_start, edge_end)) + result.append(current) + elif previous_inside: + result.append(_line_intersection(previous, current, edge_start, edge_end)) + previous = current + previous_inside = current_inside + return result + + +def _inside( + point: tuple[float, float], + edge_start: tuple[float, float], + edge_end: tuple[float, float], +) -> bool: + return _cross( + edge_end[0] - edge_start[0], + edge_end[1] - edge_start[1], + point[0] - edge_start[0], + point[1] - edge_start[1], + ) >= -_EPSILON + + +def _line_intersection( + segment_start: tuple[float, float], + segment_end: tuple[float, float], + edge_start: tuple[float, float], + edge_end: tuple[float, float], +) -> tuple[float, float]: + segment_x = segment_end[0] - segment_start[0] + segment_y = segment_end[1] - segment_start[1] + edge_x = edge_end[0] - edge_start[0] + edge_y = edge_end[1] - edge_start[1] + denominator = _cross(segment_x, segment_y, edge_x, edge_y) + if abs(denominator) <= _EPSILON: + return segment_end + offset_x = edge_start[0] - segment_start[0] + offset_y = edge_start[1] - segment_start[1] + ratio = _cross(offset_x, offset_y, edge_x, edge_y) / denominator + return ( + segment_start[0] + ratio * segment_x, + segment_start[1] + ratio * segment_y, + ) + + +def _cross(one_x: float, one_y: float, another_x: float, another_y: float) -> float: + return one_x * another_y - one_y * another_x diff --git a/src/k1link/datasets/__init__.py b/src/k1link/datasets/__init__.py index d3aeb3f..0eceef7 100644 --- a/src/k1link/datasets/__init__.py +++ b/src/k1link/datasets/__init__.py @@ -38,6 +38,14 @@ from k1link.datasets.goose_qualification import ( GroundAcceptancePolicy, qualify_goose_ground, ) +from k1link.datasets.kitti_3d_admission import ( + KITTI_3D_ADMISSION_SCHEMA, + KITTI_3D_SOURCE_ID, + Kitti3DAdmissionError, + admit_kitti_3d_object_release, + read_kitti_3d_admission, + read_kitti_standard_splits, +) from k1link.datasets.rellis_admission import ( RELLIS_ADMISSION_SCHEMA, RellisAdmissionError, @@ -79,6 +87,8 @@ __all__ = [ "GOOSE_QUALIFICATION_PREVIEW_SCHEMA", "GOOSE_QUALIFICATION_PROFILE_SCHEMA", "GOOSE_QUALIFICATION_REPORT_SCHEMA", + "KITTI_3D_ADMISSION_SCHEMA", + "KITTI_3D_SOURCE_ID", "RELLIS_CLASSES", "RELLIS_ADMISSION_SCHEMA", "RELLIS_GROUND_POLICY_SCHEMA", @@ -91,6 +101,7 @@ __all__ = [ "RellisAdmissionError", "RellisPatchworkProfile", "RellisSmokeError", + "Kitti3DAdmissionError", "GoosePatchworkProfile", "GroundAcceptancePolicy", "DegradationProfile", @@ -98,6 +109,7 @@ __all__ = [ "benchmark_goose_current_ground", "benchmark_goose_patchwork_ground", "admit_rellis_release", + "admit_kitti_3d_object_release", "build_rellis_official_smoke_preview", "calibrate_rellis_sensor_height", "configured_dataset_admission_manifest", @@ -108,6 +120,8 @@ __all__ = [ "read_dataset_admission_manifest", "read_dataset_ground_preview", "read_dataset_native_scan_preview", + "read_kitti_3d_admission", + "read_kitti_standard_splits", "read_semantic_kitti_frame", "rellis_native_scan_preview", "qualify_rellis_ground", diff --git a/src/k1link/datasets/cli.py b/src/k1link/datasets/cli.py index b2e964b..4cbef75 100644 --- a/src/k1link/datasets/cli.py +++ b/src/k1link/datasets/cli.py @@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import ( ) from k1link.datasets.goose_qualification import qualify_goose_ground from k1link.datasets.goose_review import build_goose_ground_review_pack +from k1link.datasets.kitti_3d_admission import ( + Kitti3DAdmissionError, + admit_kitti_3d_object_release, +) from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release from k1link.datasets.rellis_qualification import qualify_rellis_ground from k1link.datasets.rellis_smoke import ( @@ -28,6 +32,23 @@ app = typer.Typer( ) +@app.command("admit-kitti-3d-object") +def admit_kitti_3d_object_command( + dataset_root: Annotated[ + Path, + typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True), + ], +) -> None: + """Verify the archive-only KITTI 3D release and standard validation split.""" + + try: + manifest = admit_kitti_3d_object_release(dataset_root) + except Kitti3DAdmissionError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=2) from exc + typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True)) + + @app.command("admit-goose-validation") def admit_goose_validation_command( dataset_root: Annotated[ diff --git a/src/k1link/datasets/kitti_3d_admission.py b/src/k1link/datasets/kitti_3d_admission.py new file mode 100644 index 0000000..573efee --- /dev/null +++ b/src/k1link/datasets/kitti_3d_admission.py @@ -0,0 +1,552 @@ +"""Fail-closed admission of the KITTI 3D object development release. + +The release is admitted as independent oriented-3D-box truth for the L3 +PointPillars baseline. Source archives stay on Worker 006. This module does +not extract data, install a model, run inference, or authorize a K1 quality +claim. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import tempfile +import zipfile +from collections import Counter +from contextlib import suppress +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any, Final + +KITTI_3D_ADMISSION_SCHEMA: Final = "missioncore.kitti-3d-object-admission/v1" +KITTI_3D_SOURCE_ID: Final = "kitti-3d-object/v2017" +KITTI_3D_RELEASE_ROOT: Final = "kitti-3d-object/v2017" +KITTI_3D_LICENSE: Final = "CC-BY-NC-SA-3.0" +KITTI_3D_LICENSE_URL: Final = "https://www.cvlibs.net/datasets/kitti/index.php" +KITTI_3D_BENCHMARK_URL: Final = ( + "https://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d" +) + +KITTI_VELODYNE_ARCHIVE: Final = "data_object_velodyne.zip" +KITTI_LABEL_ARCHIVE: Final = "data_object_label_2.zip" +KITTI_CALIB_ARCHIVE: Final = "data_object_calib.zip" +KITTI_ARCHIVE_URLS: Final[dict[str, str]] = { + KITTI_VELODYNE_ARCHIVE: ( + "https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_velodyne.zip" + ), + KITTI_LABEL_ARCHIVE: ( + "https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_label_2.zip" + ), + KITTI_CALIB_ARCHIVE: ( + "https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_calib.zip" + ), +} +KITTI_ARCHIVE_BYTES: Final[dict[str, int]] = { + KITTI_VELODYNE_ARCHIVE: 28_750_710_812, + KITTI_LABEL_ARCHIVE: 5_601_213, + KITTI_CALIB_ARCHIVE: 26_854_811, +} + +KITTI_TRAINING_FRAMES: Final = 7_481 +KITTI_TEST_FRAMES: Final = 7_518 +KITTI_TARGET_CLASSES: Final = ("Car", "Pedestrian", "Cyclist") +KITTI_STANDARD_SPLIT_COMMIT: Final = ( + "233f849829b6ac19afb8af8837a0246890908755" +) +KITTI_STANDARD_SPLIT_URL: Final = ( + "https://github.com/open-mmlab/OpenPCDet/tree/" + f"{KITTI_STANDARD_SPLIT_COMMIT}/data/kitti/ImageSets" +) +KITTI_SPLIT_FILES: Final[dict[str, str]] = { + "train": "train.txt", + "validation": "val.txt", +} +KITTI_SPLIT_COUNTS: Final[dict[str, int]] = { + "train": 3_712, + "validation": 3_769, +} +KITTI_SPLIT_SHA256: Final[dict[str, str]] = { + "train": "b6417a1d9b18c8fdb085128e633d28ff321b7674a6d1b3841b8f43d865b281cb", + "validation": ( + "657ac4bcc1e156e5b106a4ca18e1f88e012787ea1d2b5d0adeea97fee903fa86" + ), +} + +MAX_ARCHIVE_ENTRIES: Final = 40_000 +MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3 +MAX_LABEL_MEMBER_BYTES: Final = 8 * 1024**2 +_FRAME_ID = re.compile(r"^[0-9]{6}$") +_VELODYNE_MEMBER = re.compile( + r"^(?Ptraining|testing)/velodyne/(?P[0-9]{6})\.bin$" +) +_LABEL_MEMBER = re.compile(r"^training/label_2/(?P[0-9]{6})\.txt$") +_CALIB_MEMBER = re.compile( + r"^(?Ptraining|testing)/calib/(?P[0-9]{6})\.txt$" +) +_CALIB_KEYS: Final = { + "P0", + "P1", + "P2", + "P3", + "R0_rect", + "Tr_velo_to_cam", + "Tr_imu_to_velo", +} + + +class Kitti3DAdmissionError(RuntimeError): + """The KITTI 3D development release violates its pinned contract.""" + + +def admit_kitti_3d_object_release( + dataset_root: Path, + *, + velodyne_archive: Path | None = None, + label_archive: Path | None = None, + calib_archive: Path | None = None, + train_split: Path | None = None, + validation_split: Path | None = None, +) -> dict[str, Any]: + """Verify the archive-only KITTI release and publish a path-free state.""" + + root = dataset_root.expanduser().absolute() + if not _is_worker_dataset_root(root): + raise Kitti3DAdmissionError( + "KITTI admission requires the canonical Worker 006 D dataset root" + ) + archive_root = root / KITTI_3D_RELEASE_ROOT / "archives" + split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / ( + f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}" + ) + archive_paths = { + KITTI_VELODYNE_ARCHIVE: _resolved_input( + archive_root, velodyne_archive, KITTI_VELODYNE_ARCHIVE + ), + KITTI_LABEL_ARCHIVE: _resolved_input( + archive_root, label_archive, KITTI_LABEL_ARCHIVE + ), + KITTI_CALIB_ARCHIVE: _resolved_input( + archive_root, calib_archive, KITTI_CALIB_ARCHIVE + ), + } + split_paths = { + "train": _resolved_input( + split_root, train_split, KITTI_SPLIT_FILES["train"] + ), + "validation": _resolved_input( + split_root, validation_split, KITTI_SPLIT_FILES["validation"] + ), + } + if any(not path.is_file() for path in (*archive_paths.values(), *split_paths.values())): + raise Kitti3DAdmissionError("one or more pinned KITTI artifacts are unavailable") + + archives: dict[str, dict[str, Any]] = {} + for filename, path in archive_paths.items(): + size_bytes = path.stat().st_size + if size_bytes != KITTI_ARCHIVE_BYTES[filename]: + raise Kitti3DAdmissionError( + f"{filename} size differs from the pinned KITTI release" + ) + archives[filename] = { + "filename": filename, + "source_url": KITTI_ARCHIVE_URLS[filename], + "size_bytes": size_bytes, + "sha256": _sha256_file(path), + "vendor_checksum_available": False, + } + + splits = _read_standard_splits(split_paths) + try: + with ( + zipfile.ZipFile(archive_paths[KITTI_VELODYNE_ARCHIVE]) as points_zip, + zipfile.ZipFile(archive_paths[KITTI_LABEL_ARCHIVE]) as labels_zip, + zipfile.ZipFile(archive_paths[KITTI_CALIB_ARCHIVE]) as calib_zip, + ): + point_members = _member_index(points_zip) + label_members = _member_index(labels_zip) + calib_members = _member_index(calib_zip) + training_points, testing_points = _validate_velodyne(point_members) + training_labels, target_counts = _validate_labels( + labels_zip, label_members + ) + training_calib, testing_calib = _validate_calibrations( + calib_zip, calib_members + ) + except (OSError, KeyError, UnicodeDecodeError, zipfile.BadZipFile) as exc: + raise Kitti3DAdmissionError("KITTI archives could not be verified") from exc + + training_ids = set(training_points) + if ( + set(training_labels) != training_ids + or set(training_calib) != training_ids + or set(testing_points) != set(testing_calib) + ): + raise Kitti3DAdmissionError("KITTI point, label, and calibration indices diverge") + split_union = set(splits["train"]) | set(splits["validation"]) + if ( + set(splits["train"]).intersection(splits["validation"]) + or split_union != training_ids + ): + raise Kitti3DAdmissionError( + "OpenPCDet train/validation split is overlapping or incomplete" + ) + validation_target_counts = _target_counts_for_frames( + archive_paths[KITTI_LABEL_ARCHIVE], + set(splits["validation"]), + ) + if any(validation_target_counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES): + raise Kitti3DAdmissionError("KITTI validation split lacks a target class") + + identity = { + "source_id": KITTI_3D_SOURCE_ID, + "archives": archives, + "split_source": { + "repository_commit": KITTI_STANDARD_SPLIT_COMMIT, + "source_url": KITTI_STANDARD_SPLIT_URL, + "sha256": KITTI_SPLIT_SHA256, + }, + "license": { + "spdx": KITTI_3D_LICENSE, + "source_url": KITTI_3D_LICENSE_URL, + "use_scope": "academic-non-commercial", + }, + "representation": { + "point_fields": ["x", "y", "z", "intensity"], + "ground_truth": "oriented-3d-boxes", + "box_coordinate_frame": "camera-rectified", + "calibration_to_sensor_frame_present": True, + }, + } + identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() + manifest = { + "schema_version": KITTI_3D_ADMISSION_SCHEMA, + "source_id": KITTI_3D_SOURCE_ID, + "observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "status": "archive-ready", + "release_identity_sha256": identity_sha256, + "storage": { + "policy": "worker-d-only", + "admitted": True, + "canonical_root": True, + "path_exposed": False, + "source_archives_extracted": False, + }, + "identity": identity, + "alignment": { + "training_frame_count": len(training_points), + "test_frame_count": len(testing_points), + "training_label_count": len(training_labels), + "training_calibration_count": len(training_calib), + "test_calibration_count": len(testing_calib), + "split_counts": KITTI_SPLIT_COUNTS, + "split_union_complete": True, + "split_overlap_count": 0, + "all_target_box_counts": dict(sorted(target_counts.items())), + "validation_target_box_counts": dict( + sorted(validation_target_counts.items()) + ), + }, + "benchmark_contract": { + "independent_ground_truth": True, + "annotations": ["oriented-3d-boxes"], + "point_fields": ["x", "y", "z", "intensity"], + "eligible_split": "validation", + "target_classes": list(KITTI_TARGET_CLASSES), + "official_test_submission_authorized": False, + "retuning_on_validation_allowed": False, + "k1_quality_claim_authorized": False, + }, + "next_action": "promote-staged-engine-then-stream-standard-validation", + } + _atomic_json(root / "state/kitti-3d-object-v2017.json", manifest) + return manifest + + +def read_kitti_3d_admission(dataset_root: Path) -> dict[str, Any]: + """Read the current path-free state and validate its content identity.""" + + root = dataset_root.expanduser().absolute() + if not _is_worker_dataset_root(root): + raise Kitti3DAdmissionError( + "KITTI admission requires the canonical Worker 006 D dataset root" + ) + path = root / "state/kitti-3d-object-v2017.json" + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Kitti3DAdmissionError("KITTI admission state is unavailable") from exc + if not isinstance(manifest, dict): + raise Kitti3DAdmissionError("KITTI admission state is not an object") + identity = manifest.get("identity") + if ( + manifest.get("schema_version") != KITTI_3D_ADMISSION_SCHEMA + or manifest.get("source_id") != KITTI_3D_SOURCE_ID + or manifest.get("status") != "archive-ready" + or not isinstance(identity, dict) + or manifest.get("release_identity_sha256") + != hashlib.sha256(_canonical_json(identity)).hexdigest() + ): + raise Kitti3DAdmissionError("KITTI admission state identity is invalid") + return manifest + + +def read_kitti_standard_splits( + dataset_root: Path, +) -> dict[str, tuple[str, ...]]: + """Read the pinned OpenPCDet split files after validating current state.""" + + root = dataset_root.expanduser().absolute() + read_kitti_3d_admission(root) + split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / ( + f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}" + ) + return _read_standard_splits( + { + split_name: split_root / filename + for split_name, filename in KITTI_SPLIT_FILES.items() + } + ) + + +def _resolved_input(root: Path, provided: Path | None, filename: str) -> Path: + return provided.expanduser().absolute() if provided is not None else root / filename + + +def _is_worker_dataset_root(root: Path) -> bool: + normalized = str(root).replace("\\", "/").rstrip("/").lower() + return normalized == "/mnt/d/ndc_missioncore/datasets" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as source: + for chunk in iter(lambda: source.read(8 * 1024**2), b""): + digest.update(chunk) + except OSError as exc: + raise Kitti3DAdmissionError("KITTI artifact cannot be hashed") from exc + return digest.hexdigest() + + +def _read_standard_splits(paths: dict[str, Path]) -> dict[str, tuple[str, ...]]: + result: dict[str, tuple[str, ...]] = {} + for split_name, path in paths.items(): + if _sha256_file(path) != KITTI_SPLIT_SHA256[split_name]: + raise Kitti3DAdmissionError( + f"KITTI {split_name} split differs from the pinned OpenPCDet commit" + ) + try: + rows = tuple( + row.strip() + for row in path.read_text(encoding="ascii").splitlines() + if row.strip() + ) + except (OSError, UnicodeDecodeError) as exc: + raise Kitti3DAdmissionError("KITTI split cannot be read") from exc + if ( + len(rows) != KITTI_SPLIT_COUNTS[split_name] + or len(set(rows)) != len(rows) + or any(_FRAME_ID.fullmatch(row) is None for row in rows) + ): + raise Kitti3DAdmissionError(f"KITTI {split_name} split is invalid") + result[split_name] = rows + return result + + +def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]: + members = source.infolist() + if not members or len(members) > MAX_ARCHIVE_ENTRIES: + raise Kitti3DAdmissionError("KITTI archive entry count is invalid") + total_uncompressed = 0 + indexed: dict[str, zipfile.ZipInfo] = {} + for member in members: + path = PurePosixPath(member.filename) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.filename + or member.file_size < 0 + or member.compress_size < 0 + ): + raise Kitti3DAdmissionError("KITTI archive contains an unsafe member") + total_uncompressed += member.file_size + if total_uncompressed > MAX_UNCOMPRESSED_BYTES: + raise Kitti3DAdmissionError("KITTI archive expands beyond the admitted limit") + if member.is_dir(): + continue + normalized = path.as_posix() + if normalized in indexed: + raise Kitti3DAdmissionError("KITTI archive contains duplicate members") + indexed[normalized] = member + return indexed + + +def _validate_velodyne( + members: dict[str, zipfile.ZipInfo], +) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]: + indexed: dict[str, dict[str, zipfile.ZipInfo]] = { + "training": {}, + "testing": {}, + } + for path, member in members.items(): + match = _VELODYNE_MEMBER.fullmatch(path) + if match is None: + continue + if member.file_size <= 0 or member.file_size % 16: + raise Kitti3DAdmissionError("KITTI Velodyne frame is not packed XYZI") + indexed[match.group("split")][match.group("frame")] = member + if ( + len(indexed["training"]) != KITTI_TRAINING_FRAMES + or len(indexed["testing"]) != KITTI_TEST_FRAMES + ): + raise Kitti3DAdmissionError("KITTI Velodyne frame count is invalid") + return indexed["training"], indexed["testing"] + + +def _validate_labels( + source: zipfile.ZipFile, + members: dict[str, zipfile.ZipInfo], +) -> tuple[dict[str, zipfile.ZipInfo], Counter[str]]: + indexed: dict[str, zipfile.ZipInfo] = {} + counts: Counter[str] = Counter() + for path, member in members.items(): + match = _LABEL_MEMBER.fullmatch(path) + if match is None: + continue + if member.file_size > MAX_LABEL_MEMBER_BYTES: + raise Kitti3DAdmissionError("KITTI label member is unexpectedly large") + frame_id = match.group("frame") + indexed[frame_id] = member + counts.update(_parse_label_member(source.read(member))) + if len(indexed) != KITTI_TRAINING_FRAMES: + raise Kitti3DAdmissionError("KITTI label frame count is invalid") + if any(counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES): + raise Kitti3DAdmissionError("KITTI release lacks a target 3D box class") + return indexed, counts + + +def _parse_label_member(payload: bytes) -> Counter[str]: + try: + text = payload.decode("ascii") + except UnicodeDecodeError as exc: + raise Kitti3DAdmissionError("KITTI label member is not ASCII") from exc + counts: Counter[str] = Counter() + for raw_line in text.splitlines(): + fields = raw_line.split() + if not fields: + continue + if len(fields) != 15: + raise Kitti3DAdmissionError("KITTI label row does not have 15 fields") + class_name = fields[0] + try: + values = [float(value) for value in fields[1:]] + except ValueError as exc: + raise Kitti3DAdmissionError("KITTI label row contains invalid numbers") from exc + if not all(math.isfinite(value) for value in values): + raise Kitti3DAdmissionError("KITTI label row contains non-finite numbers") + if class_name in KITTI_TARGET_CLASSES: + height, width, length = values[7:10] + if height <= 0 or width <= 0 or length <= 0: + raise Kitti3DAdmissionError("KITTI target box has invalid dimensions") + counts[class_name] += 1 + return counts + + +def _validate_calibrations( + source: zipfile.ZipFile, + members: dict[str, zipfile.ZipInfo], +) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]: + indexed: dict[str, dict[str, zipfile.ZipInfo]] = { + "training": {}, + "testing": {}, + } + for path, member in members.items(): + match = _CALIB_MEMBER.fullmatch(path) + if match is None: + continue + payload = source.read(member) + try: + lines = payload.decode("ascii").splitlines() + except UnicodeDecodeError as exc: + raise Kitti3DAdmissionError("KITTI calibration is not ASCII") from exc + keys: set[str] = set() + for line in lines: + if not line.strip(): + continue + key, separator, raw_values = line.partition(":") + if not separator: + raise Kitti3DAdmissionError("KITTI calibration row is invalid") + try: + values = [float(value) for value in raw_values.split()] + except ValueError as exc: + raise Kitti3DAdmissionError( + "KITTI calibration contains invalid numbers" + ) from exc + if not values or not all(math.isfinite(value) for value in values): + raise Kitti3DAdmissionError( + "KITTI calibration contains non-finite numbers" + ) + keys.add(key) + if not _CALIB_KEYS.issubset(keys): + raise Kitti3DAdmissionError("KITTI calibration lacks required transforms") + indexed[match.group("split")][match.group("frame")] = member + if ( + len(indexed["training"]) != KITTI_TRAINING_FRAMES + or len(indexed["testing"]) != KITTI_TEST_FRAMES + ): + raise Kitti3DAdmissionError("KITTI calibration frame count is invalid") + return indexed["training"], indexed["testing"] + + +def _target_counts_for_frames( + labels_archive: Path, + frame_ids: set[str], +) -> Counter[str]: + counts: Counter[str] = Counter() + try: + with zipfile.ZipFile(labels_archive) as source: + members = _member_index(source) + for frame_id in sorted(frame_ids): + path = f"training/label_2/{frame_id}.txt" + member = members.get(path) + if member is None: + raise Kitti3DAdmissionError( + "KITTI validation split references a missing label" + ) + counts.update(_parse_label_member(source.read(member))) + except (OSError, zipfile.BadZipFile) as exc: + raise Kitti3DAdmissionError( + "KITTI validation labels could not be verified" + ) from exc + return counts + + +def _canonical_json(payload: Any) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + try: + with os.fdopen(descriptor, "wb") as target: + target.write(_canonical_json(payload) + b"\n") + target.flush() + os.fsync(target.fileno()) + os.replace(temporary, path) + except BaseException: + with suppress(OSError): + os.unlink(temporary) + raise diff --git a/tests/test_kitti_3d_admission.py b/tests/test_kitti_3d_admission.py new file mode 100644 index 0000000..3568985 --- /dev/null +++ b/tests/test_kitti_3d_admission.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import hashlib +import json +import zipfile +from pathlib import Path +from typing import Any + +import pytest + +from k1link.datasets import kitti_3d_admission as module + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _calibration(*, complete: bool = True) -> str: + rows = [ + "P0: " + " ".join(["1"] * 12), + "P1: " + " ".join(["1"] * 12), + "P2: " + " ".join(["1"] * 12), + "P3: " + " ".join(["1"] * 12), + "R0_rect: " + " ".join(["1"] * 9), + "Tr_velo_to_cam: " + " ".join(["1"] * 12), + ] + if complete: + rows.append("Tr_imu_to_velo: " + " ".join(["1"] * 12)) + return "\n".join(rows) + "\n" + + +def _label(class_name: str, *, valid: bool = True) -> str: + dimensions = "1.5 1.6 3.8" if valid else "0 1.6 3.8" + return ( + f"{class_name} 0 0 0 0 0 10 10 {dimensions} 1 1 10 0\n" + ) + + +def _release( + root: Path, + monkeypatch: pytest.MonkeyPatch, + *, + labels: dict[str, str] | None = None, + complete_calibration: bool = True, + invalid_velodyne_frame: bool = False, + train_rows: tuple[str, ...] = ("000000",), + validation_rows: tuple[str, ...] = ("000001", "000002"), +) -> dict[str, Path]: + archive_root = root / module.KITTI_3D_RELEASE_ROOT / "archives" + split_root = root / module.KITTI_3D_RELEASE_ROOT / "splits" / ( + f"openpcdet-{module.KITTI_STANDARD_SPLIT_COMMIT}" + ) + archive_root.mkdir(parents=True) + split_root.mkdir(parents=True) + paths = { + module.KITTI_VELODYNE_ARCHIVE: archive_root + / module.KITTI_VELODYNE_ARCHIVE, + module.KITTI_LABEL_ARCHIVE: archive_root / module.KITTI_LABEL_ARCHIVE, + module.KITTI_CALIB_ARCHIVE: archive_root / module.KITTI_CALIB_ARCHIVE, + "train": split_root / "train.txt", + "validation": split_root / "val.txt", + } + with zipfile.ZipFile(paths[module.KITTI_VELODYNE_ARCHIVE], "w") as archive: + for frame_id in ("000000", "000001", "000002"): + size = 15 if invalid_velodyne_frame and frame_id == "000001" else 16 + archive.writestr(f"training/velodyne/{frame_id}.bin", b"\x00" * size) + for frame_id in ("000000", "000001"): + archive.writestr(f"testing/velodyne/{frame_id}.bin", b"\x00" * 16) + label_payloads = labels or { + "000000": _label("Car"), + "000001": _label("Pedestrian") + _label("Car"), + "000002": _label("Cyclist"), + } + with zipfile.ZipFile(paths[module.KITTI_LABEL_ARCHIVE], "w") as archive: + for frame_id, payload in label_payloads.items(): + archive.writestr(f"training/label_2/{frame_id}.txt", payload) + with zipfile.ZipFile(paths[module.KITTI_CALIB_ARCHIVE], "w") as archive: + for split, frames in { + "training": ("000000", "000001", "000002"), + "testing": ("000000", "000001"), + }.items(): + for frame_id in frames: + archive.writestr( + f"{split}/calib/{frame_id}.txt", + _calibration(complete=complete_calibration), + ) + paths["train"].write_text("\n".join(train_rows) + "\n", encoding="ascii") + paths["validation"].write_text( + "\n".join(validation_rows) + "\n", + encoding="ascii", + ) + monkeypatch.setattr(module, "KITTI_TRAINING_FRAMES", 3) + monkeypatch.setattr(module, "KITTI_TEST_FRAMES", 2) + monkeypatch.setattr( + module, + "KITTI_ARCHIVE_BYTES", + { + name: paths[name].stat().st_size + for name in ( + module.KITTI_VELODYNE_ARCHIVE, + module.KITTI_LABEL_ARCHIVE, + module.KITTI_CALIB_ARCHIVE, + ) + }, + ) + monkeypatch.setattr( + module, + "KITTI_SPLIT_COUNTS", + {"train": len(train_rows), "validation": len(validation_rows)}, + ) + monkeypatch.setattr( + module, + "KITTI_SPLIT_SHA256", + {"train": _sha256(paths["train"]), "validation": _sha256(paths["validation"])}, + ) + monkeypatch.setattr(module, "_is_worker_dataset_root", lambda _root: True) + return paths + + +def test_admits_archive_only_box_truth_with_path_free_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release(tmp_path, monkeypatch) + + manifest = module.admit_kitti_3d_object_release(tmp_path) + + assert manifest["status"] == "archive-ready" + assert manifest["storage"]["source_archives_extracted"] is False + assert manifest["benchmark_contract"] == { + "independent_ground_truth": True, + "annotations": ["oriented-3d-boxes"], + "point_fields": ["x", "y", "z", "intensity"], + "eligible_split": "validation", + "target_classes": ["Car", "Pedestrian", "Cyclist"], + "official_test_submission_authorized": False, + "retuning_on_validation_allowed": False, + "k1_quality_claim_authorized": False, + } + assert manifest["alignment"]["validation_target_box_counts"] == { + "Car": 1, + "Cyclist": 1, + "Pedestrian": 1, + } + serialized = json.dumps(manifest) + assert str(tmp_path) not in serialized + assert module.read_kitti_3d_admission(tmp_path) == manifest + assert module.read_kitti_standard_splits(tmp_path) == { + "train": ("000000",), + "validation": ("000001", "000002"), + } + + +def test_rejects_tampered_standard_split( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = _release(tmp_path, monkeypatch) + paths["validation"].write_text("000002\n000001\n", encoding="ascii") + + with pytest.raises(module.Kitti3DAdmissionError, match="pinned OpenPCDet"): + module.admit_kitti_3d_object_release(tmp_path) + + +def test_rejects_overlapping_split( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release( + tmp_path, + monkeypatch, + train_rows=("000000", "000001"), + validation_rows=("000001", "000002"), + ) + + with pytest.raises(module.Kitti3DAdmissionError, match="overlapping or incomplete"): + module.admit_kitti_3d_object_release(tmp_path) + + +def test_rejects_non_xyzi_velodyne_frame( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release(tmp_path, monkeypatch, invalid_velodyne_frame=True) + + with pytest.raises(module.Kitti3DAdmissionError, match="not packed XYZI"): + module.admit_kitti_3d_object_release(tmp_path) + + +def test_rejects_invalid_target_box( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release( + tmp_path, + monkeypatch, + labels={ + "000000": _label("Car"), + "000001": _label("Pedestrian", valid=False), + "000002": _label("Cyclist"), + }, + ) + + with pytest.raises(module.Kitti3DAdmissionError, match="invalid dimensions"): + module.admit_kitti_3d_object_release(tmp_path) + + +def test_rejects_missing_calibration_transform( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release(tmp_path, monkeypatch, complete_calibration=False) + + with pytest.raises(module.Kitti3DAdmissionError, match="required transforms"): + module.admit_kitti_3d_object_release(tmp_path) + + +def test_read_rejects_tampered_content_identity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _release(tmp_path, monkeypatch) + module.admit_kitti_3d_object_release(tmp_path) + state = tmp_path / "state/kitti-3d-object-v2017.json" + payload: dict[str, Any] = json.loads(state.read_text(encoding="utf-8")) + payload["identity"]["license"]["spdx"] = "unknown" + state.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(module.Kitti3DAdmissionError, match="identity is invalid"): + module.read_kitti_3d_admission(tmp_path) diff --git a/tests/test_kitti_pointpillars_benchmark.py b/tests/test_kitti_pointpillars_benchmark.py new file mode 100644 index 0000000..6b6d8aa --- /dev/null +++ b/tests/test_kitti_pointpillars_benchmark.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +import math +import zipfile +from pathlib import Path + +import pytest + +from k1link.compute.kitti_pointpillars_benchmark import ( + KittiLidarTruth, + KittiPointPillarsBenchmarkError, + PointPillarsFramePrediction, + evaluate_pointpillars_predictions, + read_kitti_validation_truth, +) +from k1link.compute.pointpillars_postprocess import PointPillarsBox + + +def _calibration() -> str: + return "\n".join( + [ + "R0_rect: 1 0 0 0 1 0 0 0 1", + "Tr_velo_to_cam: 1 0 0 0 0 1 0 0 0 0 1 0", + "", + "", + ] + ) + + +def _label(class_name: str, x_m: float) -> str: + return f"{class_name} 0 0 0 0 0 10 10 1.5 2 4 {x_m} 0 0 0\n" + + +def _truth( + frame_id: str, + class_name: str, + *, + x_m: float = 10.0, +) -> KittiLidarTruth: + return KittiLidarTruth( + frame_id=frame_id, + benchmark_class=class_name, + x_m=x_m, + y_m=0.0, + z_m=0.0, + length_m=4.0, + width_m=2.0, + height_m=1.5, + yaw_rad=0.0, + ) + + +def _prediction_box( + model_class: str, + *, + x_m: float = 10.0, + score: float = 0.9, +) -> PointPillarsBox: + return PointPillarsBox( + x_m=x_m, + y_m=0.0, + z_m=0.0, + length_m=4.0, + width_m=2.0, + height_m=1.5, + yaw_rad=0.0, + class_id={"Vehicle": 0, "Pedestrian": 1, "Cyclist": 2}[model_class], + model_class=model_class, + score=score, + ) + + +def _perfect_fixture() -> tuple[ + dict[str, tuple[KittiLidarTruth, ...]], + tuple[PointPillarsFramePrediction, ...], +]: + classes = ( + ("000000", "Car", "Vehicle"), + ("000001", "Pedestrian", "Pedestrian"), + ("000002", "Cyclist", "Cyclist"), + ) + truth = { + frame_id: (_truth(frame_id, benchmark_class),) + for frame_id, benchmark_class, _ in classes + } + predictions = tuple( + PointPillarsFramePrediction( + frame_id=frame_id, + boxes=(_prediction_box(model_class),), + inference_ms=50.0 + index, + ) + for index, (frame_id, _, model_class) in enumerate(classes) + ) + return truth, predictions + + +def test_reads_and_converts_kitti_camera_bottom_centers( + tmp_path: Path, +) -> None: + labels = tmp_path / "labels.zip" + calibrations = tmp_path / "calib.zip" + with zipfile.ZipFile(labels, "w") as archive: + archive.writestr( + "training/label_2/000000.txt", + _label("Car", 10.0), + ) + archive.writestr( + "training/label_2/000001.txt", + _label("Pedestrian", 11.0), + ) + archive.writestr( + "training/label_2/000002.txt", + _label("Cyclist", 12.0), + ) + with zipfile.ZipFile(calibrations, "w") as archive: + for frame_id in ("000000", "000001", "000002"): + archive.writestr( + f"training/calib/{frame_id}.txt", + _calibration(), + ) + + truth = read_kitti_validation_truth( + labels_archive=labels, + calibrations_archive=calibrations, + validation_frame_ids=("000000", "000001", "000002"), + ) + + car = truth["000000"][0] + assert car.x_m == 10.0 + assert car.z_m == pytest.approx(0.75) + assert car.yaw_rad == pytest.approx(-math.pi / 2.0) + assert car.length_m == 4.0 + assert car.width_m == 2.0 + + +def test_perfect_predictions_produce_complete_metrics() -> None: + truth, predictions = _perfect_fixture() + + report = evaluate_pointpillars_predictions( + truth_by_frame=truth, + predictions=predictions, + ) + + assert report["aggregates"]["bev_map40"] == pytest.approx(1.0) + assert report["aggregates"]["3d_map40"] == pytest.approx(1.0) + assert report["aggregates"]["false_occupied_rate"] == 0.0 + assert report["aggregates"]["center_error_m"]["mean"] == 0.0 + assert report["aggregates"]["range_error_m"]["mean"] == 0.0 + assert report["aggregates"]["yaw_error_rad"]["mean"] == 0.0 + assert report["aggregates"]["distance_bucket_recall"]["0-20m"]["recall"] == 1.0 + assert report["metric_contract"]["evaluation_kind"] == ( + "public-cross-domain-transfer-probe" + ) + assert report["claim_boundary"]["native_model_accuracy_evaluated"] is False + assert report["claim_boundary"]["k1_transfer_evaluated"] is False + + +def test_false_prediction_reduces_precision_and_counts_false_occupied() -> None: + truth, predictions = _perfect_fixture() + first = predictions[0] + predictions = ( + PointPillarsFramePrediction( + frame_id=first.frame_id, + boxes=( + _prediction_box("Vehicle", x_m=40.0, score=0.95), + *first.boxes, + ), + inference_ms=first.inference_ms, + ), + *predictions[1:], + ) + + report = evaluate_pointpillars_predictions( + truth_by_frame=truth, + predictions=predictions, + ) + + assert report["per_class"]["Car"]["true_positives"] == 1 + assert report["per_class"]["Car"]["false_positives"] == 1 + assert report["per_class"]["Car"]["precision"] == pytest.approx(0.5) + assert report["aggregates"]["false_occupied_rate"] == pytest.approx(0.25) + + +def test_predictions_outside_shared_cross_domain_range_are_not_false_positives() -> None: + truth, predictions = _perfect_fixture() + first = predictions[0] + predictions = ( + PointPillarsFramePrediction( + frame_id=first.frame_id, + boxes=( + _prediction_box("Vehicle", x_m=-10.0, score=0.95), + *first.boxes, + ), + inference_ms=first.inference_ms, + ), + *predictions[1:], + ) + + report = evaluate_pointpillars_predictions( + truth_by_frame=truth, + predictions=predictions, + ) + + assert report["per_class"]["Car"]["false_positives"] == 0 + assert report["aggregates"]["prediction_volume"] == { + "model_output_box_count": 4, + "evaluated_box_count": 3, + "outside_shared_range_count": 1, + } + + +def test_frame_set_must_equal_admitted_validation_split() -> None: + truth, predictions = _perfect_fixture() + + with pytest.raises( + KittiPointPillarsBenchmarkError, + match="do not equal", + ): + evaluate_pointpillars_predictions( + truth_by_frame=truth, + predictions=predictions[:-1], + ) + + +def test_unknown_model_class_is_rejected() -> None: + truth, predictions = _perfect_fixture() + first = predictions[0] + unknown = PointPillarsBox( + **{ + field: getattr(first.boxes[0], field) + for field in ( + "x_m", + "y_m", + "z_m", + "length_m", + "width_m", + "height_m", + "yaw_rad", + "class_id", + "score", + ) + }, + model_class="Unknown", + ) + predictions = ( + PointPillarsFramePrediction( + frame_id=first.frame_id, + boxes=(unknown,), + inference_ms=first.inference_ms, + ), + *predictions[1:], + ) + + with pytest.raises(KittiPointPillarsBenchmarkError, match="not admitted"): + evaluate_pointpillars_predictions( + truth_by_frame=truth, + predictions=predictions, + ) diff --git a/tests/test_l3_pointpillars_admission.py b/tests/test_l3_pointpillars_admission.py new file mode 100644 index 0000000..e431511 --- /dev/null +++ b/tests/test_l3_pointpillars_admission.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from k1link.compute.l3_pointpillars_admission import ( + L3PointPillarsAdmissionError, + build_l3_pointpillars_admission, + read_l3_pointpillars_admission, +) + +SHA = "a" * 64 +TRITON_SHA = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" + + +def _profile() -> dict[str, Any]: + return { + "schema_version": "missioncore.l3-pointpillars-benchmark-profile/v1", + "profile_id": "l3-pointpillars-public-transfer-probe-v1", + "detector": { + "family": "nvidia-tao-pointpillars", + "upstream_model_id": "nvidia/tao/pointpillarnet", + "candidate_frozen": False, + "candidate_model_version": None, + "candidate_source_sha256": None, + "candidate_label_sha256": None, + "triton_model_name": "pointpillars", + "required_source_format": "onnx", + "input_representation": "native-sensor-scan", + "input_coordinate_frame": "sensor/lidar", + "input_fields": ["x", "y", "z", "intensity"], + "batch_size": 1, + "maximum_points": 204_800, + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432, + ], + "training_domain": "proprietary-solid-state-lidar", + "training_ground_truth_publicly_reproducible": False, + "model_classes": ["Vehicle", "Pedestrian", "Cyclist"], + "onnx_contract_sha256": ( + "2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be" + ), + "postprocessing": { + "reference_repository": ( + "https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes" + ), + "reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8", + "output_row_fields": [ + "x", + "y", + "z", + "length", + "width", + "height", + "yaw", + "class_id", + "score", + ], + "class_agnostic_nms": True, + "nms_iou_threshold": 0.01, + "pre_nms_top_n": 4096, + "embedded_score_threshold": 0.1, + "embedded_contract_source": "onnx-node-attributes", + }, + }, + "runtime_policy": { + "existing_triton_only": True, + "second_serving_stack_allowed": False, + "engine_built_on_target_required": True, + "precision": "strongly-typed", + "triton_image": "nvcr.io/nvidia/tritonserver:26.06-py3", + "triton_image_digest": TRITON_SHA, + }, + "public_cross_domain_probe": { + "required_split": "validation", + "required_ground_truth": "oriented-3d-boxes", + "independent_ground_truth_required": True, + "benchmark_classes": ["Car", "Pedestrian", "Cyclist"], + "model_to_benchmark_class_mapping": { + "Vehicle": "Car", + "Pedestrian": "Pedestrian", + "Cyclist": "Cyclist", + }, + "metrics": [ + "bev-map", + "3d-map", + "center-error-m", + "range-error-m", + "yaw-error-rad", + "distance-bucket-recall", + "false-occupied-rate", + "end-to-end-latency-ms", + ], + "metric_contract": { + "official_kitti_server_metric": False, + "evaluation_kind": "public-cross-domain-transfer-probe", + "ap_interpolation": "40-point", + "difficulty_filtering": False, + "predictions_outside_shared_range_ignored": True, + "iou_thresholds": { + "Car": 0.7, + "Pedestrian": 0.5, + "Cyclist": 0.5, + }, + "distance_buckets_m": [[0, 20], [20, 40], [40, 70]], + }, + "native_accuracy_claim_allowed": False, + "retuning_allowed": False, + }, + "k1_transfer_stability": { + "requires_completed_public_cross_domain_probe": True, + "metrics": [ + "input-admission-rate", + "output-schema-valid-rate", + "deterministic-replay-rate", + "end-to-end-latency-ms", + "queue-wait-ms", + "drop-rate", + ], + "accuracy_claim_allowed": False, + "retuning_allowed": False, + }, + "authority": { + "shadow_only": True, + "commands_enabled": False, + "navigation_or_safety_accepted": False, + }, + } + + +def _semantic_dataset(dataset_id: str = "goose-3d/v2025-08-22") -> dict[str, Any]: + return { + "dataset_id": dataset_id, + "installed": True, + "release_identity_sha256": SHA, + "license": "CC-BY-SA-4.0", + "point_fields": ["x", "y", "z", "intensity"], + "annotations": ["point-semantic-labels", "point-instance-labels"], + "splits": ["validation"], + "independent_ground_truth": True, + } + + +def _box_dataset(*, installed: bool = True) -> dict[str, Any]: + return { + "dataset_id": "kitti-3d-object-detection/v1", + "installed": installed, + "release_identity_sha256": SHA if installed else None, + "license": "CC-BY-NC-SA-3.0", + "point_fields": ["x", "y", "z", "intensity"], + "annotations": ["oriented-3d-boxes"], + "splits": ["validation"], + "independent_ground_truth": True, + } + + +def _worker(*, pointpillars: bool = False) -> dict[str, Any]: + models: list[dict[str, Any]] = [ + { + "name": "yolox_s", + "backend": "onnxruntime", + "artifact_sha256": SHA, + } + ] + if pointpillars: + models.append( + { + "name": "pointpillars", + "upstream_version": "tao-6.26.03-test", + "source_model_sha256": "c" * 64, + "source_label_sha256": "e" * 64, + "source_format": "onnx", + "backend": "tensorrt", + "precision": "strongly-typed", + "artifact_sha256": "b" * 64, + "engine_built_on_target": True, + "provenance_verified": True, + "input_fields": ["x", "y", "z", "intensity"], + "maximum_points": 204_800, + "point_cloud_range": [ + -51.20000076293945, + -51.20000076293945, + -1.399999976158142, + 51.20000076293945, + 51.20000076293945, + 4.400000095367432, + ], + "model_classes": ["Vehicle", "Pedestrian", "Cyclist"], + "outputs": [ + { + "name": "output_boxes", + "dtype": "FP32", + "shape": [1, 393_216, 9], + }, + {"name": "num_boxes", "dtype": "INT32", "shape": [1]}, + ], + "representation_smoke": { + "status": "engine-executed", + "input_artifact_sha256": "d" * 64, + "input_point_count": 169_883, + "single_query_gpu_compute_ms": 55.0, + "accuracy_evaluated": False, + "navigation_or_safety_accepted": False, + }, + } + ) + return { + "schema_version": "missioncore.l3-worker-inventory/v1", + "host_id": "worker-006", + "observed_at_utc": "2026-07-30T20:57:38Z", + "serving_stack_count": 1, + "staged_models": [], + "triton": { + "container_name": "ndc-mission-core-triton", + "image": "nvcr.io/nvidia/tritonserver:26.06-py3", + "image_digest": TRITON_SHA, + "healthy": True, + "strict_readiness": True, + "model_control_mode": "explicit", + "model_repository_read_only": True, + "models": models, + }, + "gpu": { + "name": "NVIDIA GeForce RTX 4090", + "driver_version": "610.47", + "memory_total_mib": 24_564, + }, + } + + +def _write(path: Path, value: object) -> None: + path.write_text(json.dumps(value), encoding="utf-8") + + +def _build( + tmp_path: Path, + *, + profile: dict[str, Any] | None = None, + datasets: list[dict[str, Any]] | None = None, + worker: dict[str, Any] | None = None, +): + profile_path = tmp_path / "profile.json" + datasets_path = tmp_path / "datasets.json" + worker_path = tmp_path / "worker.json" + _write(profile_path, profile or _profile()) + _write( + datasets_path, + { + "schema_version": "missioncore.l3-lidar-dataset-inventory/v1", + "observed_at_utc": "2026-07-30T20:57:38Z", + "datasets": datasets or [_semantic_dataset()], + }, + ) + _write(worker_path, worker or _worker()) + return build_l3_pointpillars_admission( + profile_path=profile_path, + dataset_inventory_path=datasets_path, + worker_inventory_path=worker_path, + output_root=tmp_path / "results", + ) + + +def test_semantic_point_truth_and_missing_model_block_public_probe( + tmp_path: Path, +) -> None: + result = _build(tmp_path) + + assert result.report["status"] == "blocked-foundation-assets" + assert result.report["blocker_codes"] == [ + "public-oriented-3d-box-truth-not-admitted", + "pointpillars-compatible-candidate-not-frozen", + ] + assert result.report["next_gate"] == ( + "admit-public-3d-box-split-and-freeze-compatible-pointpillars-candidate" + ) + assert result.public_transfer_probe_authorized is False + finding = result.report["dataset_findings"][0] + assert finding["semantic_or_instance_labels_are_not_boxes"] is True + assert finding["oriented_3d_box_accuracy_eligible"] is False + assert ( + result.report["decision"]["semantic_point_labels_substitute_for_3d_boxes"] + is False + ) + assert result.report["decision"]["k1_transfer_stability_authorized"] is False + + +def test_box_truth_and_target_built_model_authorize_public_probe_only( + tmp_path: Path, +) -> None: + profile = _profile() + profile["detector"]["candidate_frozen"] = True + profile["detector"]["candidate_model_version"] = "tao-6.26.03-test" + profile["detector"]["candidate_source_sha256"] = "c" * 64 + profile["detector"]["candidate_label_sha256"] = "e" * 64 + result = _build( + tmp_path, + profile=profile, + datasets=[_semantic_dataset(), _box_dataset()], + worker=_worker(pointpillars=True), + ) + + assert result.report["status"] == "ready-for-public-cross-domain-probe" + assert result.report["blocker_codes"] == [] + assert result.report["eligible_public_probe_dataset_ids"] == [ + "kitti-3d-object-detection/v1" + ] + assert result.public_transfer_probe_authorized is True + assert result.report["decision"] == { + "public_cross_domain_probe_authorized": True, + "native_model_accuracy_claim_authorized": False, + "k1_transfer_stability_authorized": False, + "k1_transfer_quality_claim_authorized": False, + "semantic_point_labels_substitute_for_3d_boxes": False, + "fine_tuning_allowed": False, + "second_serving_stack_allowed": False, + "lab_publication_allowed": False, + "centerpoint_comparison_allowed": False, + } + assert result.report["next_gate"] == ( + "run-public-cross-domain-pointpillars-probe" + ) + + +def test_pointpillars_without_target_engine_provenance_remains_blocked( + tmp_path: Path, +) -> None: + profile = _profile() + profile["detector"]["candidate_frozen"] = True + profile["detector"]["candidate_model_version"] = "tao-6.26.03-test" + profile["detector"]["candidate_source_sha256"] = "c" * 64 + profile["detector"]["candidate_label_sha256"] = "e" * 64 + worker = _worker(pointpillars=True) + worker["triton"]["models"][1]["engine_built_on_target"] = False + + result = _build( + tmp_path, + profile=profile, + datasets=[_box_dataset()], + worker=worker, + ) + + assert result.report["blocker_codes"] == [ + "pointpillars-target-engine-or-provenance-not-verified" + ] + assert result.report["next_gate"] == "verify-target-engine-and-model-provenance" + + +def test_verified_staged_engine_is_distinct_from_live_install(tmp_path: Path) -> None: + profile = _profile() + profile["detector"]["candidate_frozen"] = True + profile["detector"]["candidate_model_version"] = "tao-6.26.03-test" + profile["detector"]["candidate_source_sha256"] = "c" * 64 + profile["detector"]["candidate_label_sha256"] = "e" * 64 + worker = _worker(pointpillars=True) + staged = worker["triton"]["models"].pop() + worker["staged_models"] = [staged] + + result = _build( + tmp_path, + profile=profile, + datasets=[_box_dataset(installed=False)], + worker=worker, + ) + + assert result.report["blocker_codes"] == [ + "public-oriented-3d-box-truth-not-admitted", + "pointpillars-model-not-installed-live", + ] + assert result.report["detector"]["staged_target_engine_ready"] is True + assert result.report["detector"]["model_ready"] is False + assert result.report["next_gate"] == ( + "admit-public-3d-box-split-then-install-staged-pointpillars-model" + ) + + +def test_second_serving_stack_is_rejected(tmp_path: Path) -> None: + profile = _profile() + profile["detector"]["candidate_frozen"] = True + profile["detector"]["candidate_model_version"] = "tao-6.26.03-test" + profile["detector"]["candidate_source_sha256"] = "c" * 64 + profile["detector"]["candidate_label_sha256"] = "e" * 64 + worker = _worker(pointpillars=True) + worker["serving_stack_count"] = 2 + + result = _build( + tmp_path, + profile=profile, + datasets=[_box_dataset()], + worker=worker, + ) + + assert result.report["blocker_codes"] == [ + "canonical-triton-runtime-policy-not-satisfied" + ] + assert result.report["runtime_checks"]["second_serving_stack_absent"] is False + assert result.report["decision"]["second_serving_stack_allowed"] is False + + +def test_profile_cannot_skip_the_public_transfer_probe(tmp_path: Path) -> None: + profile = copy.deepcopy(_profile()) + profile["k1_transfer_stability"][ + "requires_completed_public_cross_domain_probe" + ] = False + + with pytest.raises(L3PointPillarsAdmissionError): + _build(tmp_path, profile=profile) + + +def test_result_is_content_addressed_and_detects_tampering(tmp_path: Path) -> None: + first = _build(tmp_path) + second = _build(tmp_path) + assert first.result_id == second.result_id + + report_path = first.result_root / "admission-report.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["status"] = "changed" + _write(report_path, report) + with pytest.raises(L3PointPillarsAdmissionError): + read_l3_pointpillars_admission(first.result_root) diff --git a/tests/test_l3_pointpillars_worker_package.py b/tests/test_l3_pointpillars_worker_package.py new file mode 100644 index 0000000..7bdc367 --- /dev/null +++ b/tests/test_l3_pointpillars_worker_package.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import pytest + + +def _load_builder() -> ModuleType: + path = ( + Path(__file__).resolve().parents[1] + / "experiments/perception/prepare_l3_pointpillars_worker_package.py" + ) + specification = importlib.util.spec_from_file_location( + "l3_pointpillars_worker_package_test", + path, + ) + assert specification is not None and specification.loader is not None + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + return module + + +def test_builds_and_reopens_minimal_content_addressed_package( + tmp_path: Path, +) -> None: + builder = _load_builder() + repository = Path(__file__).resolve().parents[1] + admission = tmp_path / ( + "l3-pointpillars-admission-" + ("a" * 64) + ) + admission.mkdir() + (admission / "manifest.json").write_text("{}\n", encoding="utf-8") + (admission / "admission-report.json").write_text("{}\n", encoding="utf-8") + + package = builder.build_l3_worker_package( + repository_root=repository, + output_root=tmp_path / "packages", + admission_result=admission, + ) + reopened = builder.build_l3_worker_package( + repository_root=repository, + output_root=tmp_path / "packages", + admission_result=admission, + ) + manifest = builder.validate_l3_worker_package(package) + + assert reopened == package + assert manifest["package_id"] == package.name + assert manifest["identity"]["execution_policy"] == { + "sequential": True, + "parallel_workers": 1, + "existing_triton_only": True, + "container_creation_allowed": False, + "container_restart_allowed": False, + "raw_tensor_export_allowed": False, + } + assert not (package / "runtime/k1link/compute/__pycache__").exists() + assert ( + package / "runtime/k1link/datasets/kitti_3d_admission.py" + ).is_file() + + +def test_validation_rejects_modified_member(tmp_path: Path) -> None: + builder = _load_builder() + repository = Path(__file__).resolve().parents[1] + package = builder.build_l3_worker_package( + repository_root=repository, + output_root=tmp_path / "packages", + ) + target = package / "input/profile.json" + profile = json.loads(target.read_text(encoding="utf-8")) + profile["profile_id"] = "changed" + target.write_text(json.dumps(profile), encoding="utf-8") + + with pytest.raises( + builder.L3WorkerPackageError, + match="artifact changed", + ): + builder.validate_l3_worker_package(package) diff --git a/tests/test_l3_pointpillars_worker_runtime.py b/tests/test_l3_pointpillars_worker_runtime.py new file mode 100644 index 0000000..ec3cda5 --- /dev/null +++ b/tests/test_l3_pointpillars_worker_runtime.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from types import ModuleType + +import numpy as np +import pytest + + +def _load_worker() -> ModuleType: + path = ( + Path(__file__).resolve().parents[1] + / "experiments/perception/worker/run_l3_pointpillars_public_baseline.py" + ) + specification = importlib.util.spec_from_file_location( + "l3_pointpillars_worker_test", + path, + ) + assert specification is not None and specification.loader is not None + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + return module + + +class _Response: + def __init__(self, payload: bytes, header_length: int) -> None: + self._payload = payload + self.headers = {"Inference-Header-Content-Length": str(header_length)} + self.status = 200 + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return self._payload + + +def test_binary_triton_response_parses_both_outputs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + worker = _load_worker() + boxes = np.zeros((1, 393_216, 9), dtype=np.float32) + boxes[0, 0] = np.asarray( + [10.0, 0.0, 0.0, 4.0, 2.0, 1.5, 0.0, 0.0, 0.9], + dtype=np.float32, + ) + count = np.asarray([1], dtype=np.int32) + count_bytes = count.tobytes() + boxes_bytes = boxes.tobytes() + header = json.dumps( + { + "outputs": [ + { + "name": "num_boxes", + "datatype": "INT32", + "shape": [1], + "parameters": {"binary_data_size": len(count_bytes)}, + }, + { + "name": "output_boxes", + "datatype": "FP32", + "shape": [1, 393_216, 9], + "parameters": {"binary_data_size": len(boxes_bytes)}, + }, + ] + }, + separators=(",", ":"), + ).encode() + payload = header + count_bytes + boxes_bytes + monkeypatch.setattr( + worker.urllib.request, + "urlopen", + lambda *_args, **_kwargs: _Response(payload, len(header)), + ) + + output_boxes, output_count, elapsed_ms = worker._infer( + "http://127.0.0.1:8000", + np.zeros((1, 204_800, 4), dtype=np.float32), + np.asarray([1], dtype=np.int32), + ) + + assert output_boxes.shape == (1, 393_216, 9) + assert output_count.tolist() == [1] + assert output_boxes[0, 0, 8] == pytest.approx(0.9) + assert elapsed_ms > 0.0 diff --git a/tests/test_pointpillars_postprocess.py b/tests/test_pointpillars_postprocess.py new file mode 100644 index 0000000..a7881d4 --- /dev/null +++ b/tests/test_pointpillars_postprocess.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from k1link.compute.pointpillars_postprocess import ( + PointPillarsBox, + PointPillarsPostprocessError, + decode_pointpillars_output, + oriented_3d_iou, + oriented_bev_iou, +) + + +def _outputs(rows: list[list[float]]) -> tuple[np.ndarray, np.ndarray]: + boxes = np.zeros((1, 393_216, 9), dtype=np.float32) + boxes[0, : len(rows)] = np.asarray(rows, dtype=np.float32) + return boxes, np.asarray([len(rows)], dtype=np.int32) + + +def _row( + *, + x: float, + y: float = 0.0, + length: float = 4.0, + width: float = 2.0, + yaw: float = 0.0, + class_id: int = 0, + score: float = 0.9, +) -> list[float]: + return [x, y, 0.0, length, width, 1.5, yaw, float(class_id), score] + + +def _box(*, yaw: float = 0.0, x: float = 0.0) -> PointPillarsBox: + return PointPillarsBox( + x_m=x, + y_m=0.0, + z_m=0.0, + length_m=4.0, + width_m=2.0, + height_m=1.5, + yaw_rad=yaw, + class_id=0, + model_class="Vehicle", + score=0.9, + ) + + +def test_decodes_native_label_order_and_sorts_by_score() -> None: + output_boxes, num_boxes = _outputs( + [ + _row(x=20.0, class_id=2, score=0.6), + _row(x=0.0, class_id=0, score=0.9), + _row(x=10.0, class_id=1, score=0.8), + ] + ) + + decoded = decode_pointpillars_output(output_boxes, num_boxes) + + assert [box.model_class for box in decoded] == [ + "Vehicle", + "Pedestrian", + "Cyclist", + ] + assert [box.score for box in decoded] == pytest.approx([0.9, 0.8, 0.6]) + + +def test_nms_reproduces_nvidia_sample_class_agnostic_suppression() -> None: + output_boxes, num_boxes = _outputs( + [ + _row(x=0.0, class_id=0, score=0.9), + _row(x=0.1, class_id=1, score=0.8), + _row(x=20.0, class_id=1, score=0.7), + ] + ) + + decoded = decode_pointpillars_output(output_boxes, num_boxes) + + assert [(box.x_m, box.model_class) for box in decoded] == [ + (0.0, "Vehicle"), + (20.0, "Pedestrian"), + ] + + +def test_pre_nms_cap_is_applied_after_stable_score_ordering() -> None: + output_boxes, num_boxes = _outputs( + [ + _row(x=0.0, score=0.7), + _row(x=10.0, score=0.9), + _row(x=20.0, score=0.8), + ] + ) + + decoded = decode_pointpillars_output( + output_boxes, + num_boxes, + pre_nms_top_n=2, + ) + + assert [box.x_m for box in decoded] == [10.0, 20.0] + + +def test_oriented_bev_iou_handles_rotation_and_separation() -> None: + assert oriented_bev_iou(_box(), _box()) == pytest.approx(1.0) + assert oriented_bev_iou(_box(), _box(yaw=math.pi / 2.0)) == pytest.approx( + 1.0 / 3.0 + ) + assert oriented_bev_iou(_box(), _box(x=20.0)) == 0.0 + assert oriented_3d_iou(_box(), _box()) == pytest.approx(1.0) + assert oriented_3d_iou(_box(), _box(x=20.0)) == 0.0 + + +@pytest.mark.parametrize( + ("row", "message"), + [ + (_row(x=0.0, class_id=3), "class id"), + (_row(x=0.0, length=0.0), "dimensions or score"), + (_row(x=0.0, score=0.09), "dimensions or score"), + (_row(x=0.0, score=1.1), "dimensions or score"), + ], +) +def test_invalid_candidate_fails_closed(row: list[float], message: str) -> None: + output_boxes, num_boxes = _outputs([row]) + + with pytest.raises(PointPillarsPostprocessError, match=message): + decode_pointpillars_output(output_boxes, num_boxes) + + +def test_output_tensor_contract_is_exact() -> None: + with pytest.raises(PointPillarsPostprocessError, match="output_boxes contract"): + decode_pointpillars_output( + np.zeros((1, 1, 9), dtype=np.float32), + np.asarray([0], dtype=np.int32), + )