SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): add PointPillars transfer gate

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-31 10:48:27 +03:00
parent 29fee5c0ac
commit a9a5ca5a26
23 changed files with 5407 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/l3_dataset_inventory_2026-07-30.json
+67
View File
@@ -0,0 +1,67 @@
{
"datasets": [
{
"annotations": [
"point-semantic-labels",
"point-instance-labels"
],
"dataset_id": "goose-3d/v2025-08-22",
"independent_ground_truth": true,
"installed": true,
"license": "CC-BY-SA-4.0",
"point_fields": [
"x",
"y",
"z",
"intensity"
],
"release_identity_sha256": "0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f",
"splits": [
"validation"
]
},
{
"annotations": [
"point-semantic-labels",
"point-instance-labels"
],
"dataset_id": "rellis-3d/v1.1",
"independent_ground_truth": true,
"installed": true,
"license": "CC-BY-NC-SA-3.0",
"point_fields": [
"x",
"y",
"z",
"intensity"
],
"release_identity_sha256": "a3f3f161a5a7edccdf66cea75ecf9004b8f6a895282faabe661c075c15a82774",
"splits": [
"train",
"validation",
"test"
]
},
{
"annotations": [
"oriented-3d-boxes"
],
"dataset_id": "kitti-3d-object/v2017",
"independent_ground_truth": true,
"installed": true,
"license": "CC-BY-NC-SA-3.0",
"point_fields": [
"x",
"y",
"z",
"intensity"
],
"release_identity_sha256": "2c9615bedca56b492b204b614d4419db6431626e2a6867e207a95999beefcf47",
"splits": [
"validation"
]
}
],
"observed_at_utc": "2026-07-31T06:45:42.249612Z",
"schema_version": "missioncore.l3-lidar-dataset-inventory/v1"
}
experiments/perception/l3_pointpillars_benchmark_profile.json
+139
View File
@@ -0,0 +1,139 @@
{
"authority": {
"commands_enabled": false,
"navigation_or_safety_accepted": false,
"shadow_only": true
},
"detector": {
"batch_size": 1,
"candidate_frozen": true,
"candidate_model_version": "deployable_v1.1",
"candidate_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da",
"candidate_source_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1",
"model_classes": [
"Vehicle",
"Pedestrian",
"Cyclist"
],
"onnx_contract_sha256": "2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be",
"family": "nvidia-tao-pointpillars",
"input_coordinate_frame": "sensor/lidar",
"input_fields": [
"x",
"y",
"z",
"intensity"
],
"input_representation": "native-sensor-scan",
"maximum_points": 204800,
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432
],
"postprocessing": {
"class_agnostic_nms": true,
"embedded_contract_source": "onnx-node-attributes",
"embedded_score_threshold": 0.1,
"nms_iou_threshold": 0.01,
"output_row_fields": [
"x",
"y",
"z",
"length",
"width",
"height",
"yaw",
"class_id",
"score"
],
"pre_nms_top_n": 4096,
"reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8",
"reference_repository": "https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes"
},
"required_source_format": "onnx",
"training_domain": "proprietary-solid-state-lidar",
"training_ground_truth_publicly_reproducible": false,
"triton_model_name": "pointpillars",
"upstream_model_id": "nvidia/tao/pointpillarnet"
},
"k1_transfer_stability": {
"accuracy_claim_allowed": false,
"metrics": [
"input-admission-rate",
"output-schema-valid-rate",
"deterministic-replay-rate",
"end-to-end-latency-ms",
"queue-wait-ms",
"drop-rate"
],
"requires_completed_public_cross_domain_probe": true,
"retuning_allowed": false
},
"public_cross_domain_probe": {
"benchmark_classes": [
"Car",
"Pedestrian",
"Cyclist"
],
"independent_ground_truth_required": true,
"metrics": [
"bev-map",
"3d-map",
"center-error-m",
"range-error-m",
"yaw-error-rad",
"distance-bucket-recall",
"false-occupied-rate",
"end-to-end-latency-ms"
],
"metric_contract": {
"ap_interpolation": "40-point",
"difficulty_filtering": false,
"distance_buckets_m": [
[
0,
20
],
[
20,
40
],
[
40,
70
]
],
"iou_thresholds": {
"Car": 0.7,
"Cyclist": 0.5,
"Pedestrian": 0.5
},
"evaluation_kind": "public-cross-domain-transfer-probe",
"official_kitti_server_metric": false,
"predictions_outside_shared_range_ignored": true
},
"model_to_benchmark_class_mapping": {
"Cyclist": "Cyclist",
"Pedestrian": "Pedestrian",
"Vehicle": "Car"
},
"native_accuracy_claim_allowed": false,
"required_ground_truth": "oriented-3d-boxes",
"required_split": "validation",
"retuning_allowed": false
},
"profile_id": "l3-pointpillars-public-transfer-probe-v1",
"runtime_policy": {
"engine_built_on_target_required": true,
"existing_triton_only": true,
"precision": "strongly-typed",
"second_serving_stack_allowed": false,
"triton_image": "nvcr.io/nvidia/tritonserver:26.06-py3",
"triton_image_digest": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
},
"schema_version": "missioncore.l3-pointpillars-benchmark-profile/v1"
}
experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json
+125
View File
@@ -0,0 +1,125 @@
{
"schema_version": "missioncore.l3-pointpillars-onnx-contract/v1",
"source": {
"model_id": "nvidia/tao/pointpillarnet",
"model_version": "deployable_v1.1",
"onnx_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1"
},
"extraction": {
"onnx_version": "1.22.0",
"method": "read-only ONNX graph attribute inspection",
"observed_at_utc": "2026-07-31T07:16:00Z"
},
"inputs": [
{
"dtype": "FP32",
"name": "points",
"shape": [
"batch",
204800,
4
]
},
{
"dtype": "INT32",
"name": "num_points",
"shape": [
"batch"
]
}
],
"outputs": [
{
"dtype": "FP32",
"name": "output_boxes",
"shape": [
"batch",
393216,
9
]
},
{
"dtype": "INT32",
"name": "num_boxes",
"shape": [
"batch"
]
}
],
"plugins": {
"DecodeBbox3DPlugin": {
"anchor_bottom_height": [
-1.7799999713897705,
-0.6000000238418579,
-0.6000000238418579
],
"anchors": [
3.9000000953674316,
1.600000023841858,
1.559999942779541,
0.0,
3.9000000953674316,
1.600000023841858,
1.559999942779541,
1.5700000524520874,
0.800000011920929,
0.6000000238418579,
1.7300000190734863,
0.0,
0.800000011920929,
0.6000000238418579,
1.7300000190734863,
1.5700000524520874,
1.7599999904632568,
0.6000000238418579,
1.7300000190734863,
0.0,
1.7599999904632568,
0.6000000238418579,
1.7300000190734863,
1.5700000524520874
],
"dir_limit_offset": 0.0,
"dir_offset": 0.7853900194168091,
"num_dir_bins": 2,
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432
],
"score_threshold": 0.10000000149011612
},
"PillarScatterPlugin": {
"dense_shape": [
512,
512
]
},
"VoxelGeneratorPlugin": {
"max_num_points_per_voxel": 32,
"max_voxels": 10000,
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432
],
"voxel_feature_num": 10,
"voxel_size": [
0.20000000298023224,
0.20000000298023224,
5.800000190734863
]
}
},
"claim_boundary": {
"model_card_training_domain": "proprietary-solid-state-lidar",
"public_training_truth_available": false,
"public_kitti_is_cross_domain": true
}
}
experiments/perception/l3_worker_inventory_2026-07-30.json
+163
View File
@@ -0,0 +1,163 @@
{
"gpu": {
"driver_version": "610.47",
"memory_total_mib": 24564,
"name": "NVIDIA GeForce RTX 4090"
},
"host_id": "worker-006",
"observed_at_utc": "2026-07-31T06:50:46.660883Z",
"schema_version": "missioncore.l3-worker-inventory/v1",
"serving_stack_count": 1,
"staged_models": [
{
"artifact_sha256": "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481",
"backend": "tensorrt",
"model_classes": [
"Vehicle",
"Pedestrian",
"Cyclist"
],
"engine_built_on_target": true,
"input_fields": [
"x",
"y",
"z",
"intensity"
],
"maximum_points": 204800,
"name": "pointpillars",
"outputs": [
{
"dtype": "FP32",
"name": "output_boxes",
"shape": [
1,
393216,
9
]
},
{
"dtype": "INT32",
"name": "num_boxes",
"shape": [
1
]
}
],
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432
],
"precision": "strongly-typed",
"provenance_verified": true,
"representation_smoke": {
"accuracy_evaluated": false,
"input_artifact_sha256": "95be16aec8496260ec25ff716c79d343a3fd9e3f4957df99bb8007c6b21c9f59",
"input_point_count": 169883,
"navigation_or_safety_accepted": false,
"single_query_gpu_compute_ms": 55.2069,
"source_frame_id": "2022-07-22_flight__0071_1658494234334310308",
"source_id": "goose-3d/v2025-08-22",
"status": "engine-executed"
},
"source_format": "onnx",
"source_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da",
"source_model_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1",
"upstream_version": "deployable_v1.1"
}
],
"triton": {
"container_name": "ndc-mission-core-triton",
"healthy": true,
"image": "nvcr.io/nvidia/tritonserver:26.06-py3",
"image_digest": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
"model_control_mode": "explicit",
"model_repository_read_only": true,
"models": [
{
"artifact_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
"backend": "onnxruntime",
"name": "yolox_s"
},
{
"artifact_sha256": "12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481",
"backend": "tensorrt",
"engine_built_on_target": true,
"input_fields": [
"x",
"y",
"z",
"intensity"
],
"live_schema_smoke": {
"accuracy_evaluated": false,
"inference_ms": 106.79901000548853,
"input_artifact_sha256": "59a02fdaaab3b7e903713cb618e8f53efcaf71c144436ddfcdf4f28bdbd73d20",
"input_point_count": 120268,
"observed_model_classes": [
"Cyclist",
"Vehicle"
],
"output_box_count": 58,
"source_frame_id": "000001",
"source_id": "kitti-3d-object/v2017",
"status": "engine-schema-executed"
},
"maximum_points": 204800,
"model_classes": [
"Vehicle",
"Pedestrian",
"Cyclist"
],
"name": "pointpillars",
"outputs": [
{
"dtype": "FP32",
"name": "output_boxes",
"shape": [
1,
393216,
9
]
},
{
"dtype": "INT32",
"name": "num_boxes",
"shape": [
1
]
}
],
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432
],
"precision": "strongly-typed",
"provenance_verified": true,
"representation_smoke": {
"accuracy_evaluated": false,
"input_artifact_sha256": "95be16aec8496260ec25ff716c79d343a3fd9e3f4957df99bb8007c6b21c9f59",
"input_point_count": 169883,
"navigation_or_safety_accepted": false,
"single_query_gpu_compute_ms": 55.2069,
"source_frame_id": "2022-07-22_flight__0071_1658494234334310308",
"source_id": "goose-3d/v2025-08-22",
"status": "engine-executed"
},
"source_format": "onnx",
"source_label_sha256": "0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da",
"source_model_sha256": "2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1",
"upstream_version": "deployable_v1.1"
}
],
"strict_readiness": true
}
}
experiments/perception/prepare_l3_pointpillars_worker_package.py
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""Build the minimal immutable L3 PointPillars package for Worker 006."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCHEMA = "missioncore.l3-pointpillars-worker-package/v1"
_SOURCE_FILES = {
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
"runtime/k1link/artifacts.py": "src/k1link/artifacts.py",
"runtime/k1link/compute/kitti_pointpillars_benchmark.py": (
"src/k1link/compute/kitti_pointpillars_benchmark.py"
),
"runtime/k1link/compute/l3_pointpillars_admission.py": (
"src/k1link/compute/l3_pointpillars_admission.py"
),
"runtime/k1link/compute/pointpillars_postprocess.py": (
"src/k1link/compute/pointpillars_postprocess.py"
),
"runtime/k1link/datasets/kitti_3d_admission.py": (
"src/k1link/datasets/kitti_3d_admission.py"
),
"runtime/install_l3_pointpillars_triton_model.py": (
"experiments/perception/worker/install_l3_pointpillars_triton_model.py"
),
"runtime/admit_l3_kitti_release.py": (
"experiments/perception/worker/admit_l3_kitti_release.py"
),
"runtime/run_l3_pointpillars_admission.py": (
"experiments/perception/run_l3_pointpillars_admission.py"
),
"runtime/run_l3_pointpillars_public_baseline.py": (
"experiments/perception/worker/run_l3_pointpillars_public_baseline.py"
),
"runtime/smoke_l3_pointpillars_live.py": (
"experiments/perception/worker/smoke_l3_pointpillars_live.py"
),
}
_GENERATED_FILES = {
"runtime/k1link/compute/__init__.py": (
'"""Minimal L3 worker projection; import compute modules explicitly."""\n'
),
"runtime/k1link/datasets/__init__.py": (
'"""Minimal L3 worker projection; import KITTI admission explicitly."""\n'
),
}
_INPUT_FILES = {
"input/profile.json": "experiments/perception/l3_pointpillars_benchmark_profile.json",
"input/onnx-contract.json": (
"experiments/perception/l3_pointpillars_onnx_contract_2026-07-31.json"
),
"input/dataset-inventory.json": (
"experiments/perception/l3_dataset_inventory_2026-07-30.json"
),
"input/worker-inventory.json": (
"experiments/perception/l3_worker_inventory_2026-07-30.json"
),
"input/triton/config.pbtxt": "experiments/perception/triton/pointpillars/config.pbtxt",
}
class L3WorkerPackageError(RuntimeError):
"""The L3 worker package source or immutable package is invalid."""
def build_l3_worker_package(
*,
repository_root: Path,
output_root: Path,
admission_result: Path | None = None,
) -> Path:
"""Build or reopen a content-addressed minimal Worker 006 runtime."""
repository = repository_root.resolve(strict=True)
sources: dict[str, Path | None] = {
target: repository / relative for target, relative in _SOURCE_FILES.items()
}
sources.update(
{
target: repository / relative
for target, relative in _INPUT_FILES.items()
}
)
sources.update({target: None for target in _GENERATED_FILES})
if admission_result is not None:
result = admission_result.resolve(strict=True)
if not result.is_dir() or result.is_symlink():
raise L3WorkerPackageError("L3 admission result root is invalid")
admission_root = f"input/admission/{result.name}"
sources[f"{admission_root}/manifest.json"] = result / "manifest.json"
sources[f"{admission_root}/admission-report.json"] = (
result / "admission-report.json"
)
descriptors: list[dict[str, Any]] = []
for relative, source in sorted(sources.items()):
payload = _payload(relative, source)
descriptors.append(
{
"path": relative,
"byte_length": len(payload),
"sha256": hashlib.sha256(payload).hexdigest(),
}
)
identity = {
"schema_version": SCHEMA,
"classification": "minimal-l3-pointpillars-worker-runtime",
"worker_host_id": "worker-006",
"source_artifacts": descriptors,
"artifact_paths": [row["path"] for row in descriptors],
"execution_policy": {
"sequential": True,
"parallel_workers": 1,
"existing_triton_only": True,
"container_creation_allowed": False,
"container_restart_allowed": False,
"raw_tensor_export_allowed": False,
},
"authority": {
"shadow_only": True,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
package_id = f"l3-pointpillars-worker-package-{identity_sha256}"
output = output_root.expanduser().absolute()
output.mkdir(mode=0o700, parents=True, exist_ok=True)
destination = output / package_id
if destination.exists():
validate_l3_worker_package(destination)
return destination
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
try:
for relative, source in sources.items():
target = staging / relative
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
if source is None:
target.write_text(_GENERATED_FILES[relative], encoding="utf-8")
else:
shutil.copyfile(source, target)
artifacts = [
{
"kind": relative,
"path": relative,
"byte_length": (staging / relative).stat().st_size,
"sha256": _sha256(staging / relative),
}
for relative in sorted(sources)
]
manifest = {
"schema_version": SCHEMA,
"package_id": package_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": datetime.now(UTC)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z"),
"artifacts": artifacts,
}
_write_json(staging / "manifest.json", manifest)
validate_l3_worker_package(staging, allow_staging=True)
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
validate_l3_worker_package(destination)
return destination
def validate_l3_worker_package(
root: Path,
*,
allow_staging: bool = False,
) -> dict[str, Any]:
"""Validate package identity, exact file set, and every member digest."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / "manifest.json")
identity = manifest.get("identity")
identity_sha256 = manifest.get("identity_sha256")
package_id = manifest.get("package_id")
artifacts = manifest.get("artifacts")
expected_name = (
isinstance(package_id, str)
and (
resolved.name == package_id
or (
allow_staging
and resolved.name.startswith(f".{package_id}.")
and resolved.name.endswith(".tmp")
)
)
)
if (
manifest.get("schema_version") != SCHEMA
or not isinstance(identity, dict)
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or package_id != f"l3-pointpillars-worker-package-{identity_sha256}"
or not expected_name
or not isinstance(artifacts, list)
):
raise L3WorkerPackageError("L3 worker package identity is invalid")
expected_paths = set(identity.get("artifact_paths", []))
actual_paths = {
path.relative_to(resolved).as_posix()
for path in resolved.rglob("*")
if path.is_file()
}
if (
not expected_paths
or actual_paths != expected_paths | {"manifest.json"}
or len(artifacts) != len(expected_paths)
):
raise L3WorkerPackageError("L3 worker package file set changed")
observed: set[str] = set()
for row in artifacts:
if not isinstance(row, dict):
raise L3WorkerPackageError("L3 worker package artifact is invalid")
relative = row.get("path")
path = resolved / str(relative)
if (
not isinstance(relative, str)
or relative not in expected_paths
or relative in observed
or Path(relative).is_absolute()
or ".." in Path(relative).parts
or not path.is_file()
or path.is_symlink()
or row.get("kind") != relative
or row.get("byte_length") != path.stat().st_size
or row.get("sha256") != _sha256(path)
):
raise L3WorkerPackageError("L3 worker package artifact changed")
observed.add(relative)
if observed != expected_paths:
raise L3WorkerPackageError("L3 worker package artifact coverage changed")
return manifest
def _payload(relative: str, source: Path | None) -> bytes:
if source is None:
return _GENERATED_FILES[relative].encode()
if not source.is_file() or source.is_symlink():
raise L3WorkerPackageError(f"L3 package source is invalid: {relative}")
return source.read_bytes()
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, json.JSONDecodeError) as exc:
raise L3WorkerPackageError("L3 worker package manifest is invalid") from exc
if not isinstance(value, dict):
raise L3WorkerPackageError("L3 worker package manifest is invalid")
return value
def _write_json(path: Path, value: object) -> None:
with path.open("x", encoding="utf-8", newline="\n") as stream:
json.dump(value, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--repository-root", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--admission-result", type=Path)
args = parser.parse_args()
package = build_l3_worker_package(
repository_root=args.repository_root,
output_root=args.output_root,
admission_result=args.admission_result,
)
print(package)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_l3_pointpillars_admission.py
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Build the immutable L3 NVIDIA PointPillars benchmark admission."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.l3_pointpillars_admission import (
build_l3_pointpillars_admission,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--dataset-inventory", type=Path, required=True)
parser.add_argument("--worker-inventory", type=Path, required=True)
parser.add_argument(
"--output-root",
type=Path,
default=Path(".runtime/compute-experiments/l3/pointpillars-admissions"),
)
args = parser.parse_args()
result = build_l3_pointpillars_admission(
profile_path=args.profile,
dataset_inventory_path=args.dataset_inventory,
worker_inventory_path=args.worker_inventory,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"status": result.report["status"],
"blocker_codes": result.report["blocker_codes"],
"next_gate": result.report["next_gate"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0 if result.public_transfer_probe_authorized else 3
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/triton/pointpillars/config.pbtxt
+42
View File
@@ -0,0 +1,42 @@
name: "pointpillars"
platform: "tensorrt_plan"
max_batch_size: 0
input [
{
name: "points"
data_type: TYPE_FP32
dims: [1, 204800, 4]
},
{
name: "num_points"
data_type: TYPE_INT32
dims: [1]
}
]
output [
{
name: "output_boxes"
data_type: TYPE_FP32
dims: [1, 393216, 9]
},
{
name: "num_boxes"
data_type: TYPE_INT32
dims: [1]
}
]
instance_group [
{
count: 1
kind: KIND_GPU
}
]
version_policy {
specific {
versions: 1
}
}
experiments/perception/worker/admit_l3_kitti_release.py
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Admit the canonical archive-only KITTI release on Worker 006."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.datasets.kitti_3d_admission import (
admit_kitti_3d_object_release,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--dataset-root",
type=Path,
default=Path("/mnt/d/NDC_MISSIONCORE/datasets"),
)
args = parser.parse_args()
result = admit_kitti_3d_object_release(args.dataset_root)
print(
json.dumps(
{
"source_id": result["source_id"],
"status": result["status"],
"release_identity_sha256": result["release_identity_sha256"],
"training_frame_count": result["alignment"][
"training_frame_count"
],
"validation_frame_count": result["alignment"]["split_counts"][
"validation"
],
"next_action": result["next_action"],
},
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/install_l3_pointpillars_triton_model.py
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""Promote the verified PointPillars engine into canonical Triton exactly once."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import tempfile
import urllib.request
from contextlib import suppress
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from k1link.compute.l3_pointpillars_admission import (
read_l3_pointpillars_admission,
)
PROVENANCE_SCHEMA = "missioncore.triton-model-provenance/v1"
MODEL_NAME = "pointpillars"
EXPECTED_ENGINE_SHA256 = (
"12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481"
)
EXPECTED_ENGINE_BYTES = 8_785_436
EXPECTED_CONFIG_SHA256 = (
"a68e7e37ae611b7d4c3fb633b31360ddbdf26ab0a37a363cc566357e614a385c"
)
EXPECTED_SOURCE_SHA256 = (
"2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1"
)
EXPECTED_LABEL_SHA256 = (
"0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da"
)
CANONICAL_MODEL_REPOSITORY = Path("/mnt/d/NDC_MISSIONCORE/runtime/models")
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--admission-result", type=Path, required=True)
parser.add_argument("--engine", type=Path, required=True)
parser.add_argument("--config", type=Path, required=True)
parser.add_argument(
"--model-repository",
type=Path,
default=CANONICAL_MODEL_REPOSITORY,
)
parser.add_argument("--triton-url", default="http://127.0.0.1:8000")
args = parser.parse_args()
admission = read_l3_pointpillars_admission(args.admission_result)
if (
admission.report.get("status") != "blocked-foundation-assets"
or admission.report.get("blocker_codes")
!= ["pointpillars-model-not-installed-live"]
or admission.report.get("eligible_public_probe_dataset_ids")
!= ["kitti-3d-object/v2017"]
or admission.report.get("detector", {}).get("staged_target_engine_ready")
is not True
):
raise RuntimeError(
"L3 admission does not authorize canonical PointPillars promotion"
)
engine = args.engine.resolve(strict=True)
config = args.config.resolve(strict=True)
if engine.stat().st_size != EXPECTED_ENGINE_BYTES or _sha256(
engine
) != EXPECTED_ENGINE_SHA256:
raise RuntimeError("PointPillars target engine identity changed")
if _sha256(config) != EXPECTED_CONFIG_SHA256:
raise RuntimeError("PointPillars Triton configuration identity changed")
repository = args.model_repository.expanduser().absolute()
if repository != CANONICAL_MODEL_REPOSITORY or not repository.is_dir():
raise RuntimeError("PointPillars promotion requires the canonical model repository")
destination = repository / MODEL_NAME
if destination.exists():
_validate_existing(destination)
_require_model_ready(args.triton_url)
print(
json.dumps(
{"status": "already-installed", "model": MODEL_NAME},
sort_keys=True,
)
)
return 0
provenance = {
"schema_version": PROVENANCE_SCHEMA,
"model_name": MODEL_NAME,
"upstream_model_id": "nvidia/tao/pointpillarnet",
"upstream_version": "deployable_v1.1",
"source_format": "onnx",
"source_model_sha256": EXPECTED_SOURCE_SHA256,
"source_label_sha256": EXPECTED_LABEL_SHA256,
"engine_sha256": EXPECTED_ENGINE_SHA256,
"engine_byte_length": EXPECTED_ENGINE_BYTES,
"config_sha256": EXPECTED_CONFIG_SHA256,
"engine_built_on_target": True,
"worker_host_id": "worker-006",
"gpu": "NVIDIA GeForce RTX 4090",
"compute_capability": "8.9",
"tensorrt_version": "11.0.0",
"precision": "strongly-typed",
"admission_result_id": admission.result_id,
"installed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"authority": {
"shadow_only": True,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
staging = Path(
tempfile.mkdtemp(
dir=repository,
prefix=f".{MODEL_NAME}.",
suffix=".incomplete",
)
)
try:
version = staging / "1"
version.mkdir(mode=0o700)
shutil.copyfile(engine, version / "model.plan")
shutil.copyfile(config, staging / "config.pbtxt")
_atomic_json(staging / "provenance.json", provenance)
if (
_sha256(version / "model.plan") != EXPECTED_ENGINE_SHA256
or _sha256(staging / "config.pbtxt") != EXPECTED_CONFIG_SHA256
):
raise RuntimeError("PointPillars staged model package changed during copy")
os.replace(staging, destination)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
request = urllib.request.Request(
f"{args.triton_url.rstrip('/')}/v2/repository/models/{MODEL_NAME}/load",
data=b"{}",
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=180) as response:
if response.status != 200:
raise RuntimeError("canonical Triton rejected PointPillars load")
_require_model_ready(args.triton_url)
except OSError as exc:
raise RuntimeError(
"PointPillars package is installed but canonical Triton did not load it"
) from exc
print(
json.dumps(
{
"status": "installed-and-ready",
"model": MODEL_NAME,
"engine_sha256": EXPECTED_ENGINE_SHA256,
"provenance_sha256": _sha256(destination / "provenance.json"),
},
sort_keys=True,
)
)
return 0
def _validate_existing(destination: Path) -> None:
engine = destination / "1/model.plan"
config = destination / "config.pbtxt"
provenance_path = destination / "provenance.json"
if (
not engine.is_file()
or not config.is_file()
or not provenance_path.is_file()
or engine.stat().st_size != EXPECTED_ENGINE_BYTES
or _sha256(engine) != EXPECTED_ENGINE_SHA256
or _sha256(config) != EXPECTED_CONFIG_SHA256
):
raise RuntimeError("existing PointPillars model package is not admitted")
try:
provenance = json.loads(provenance_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError("existing PointPillars provenance is invalid") from exc
if (
not isinstance(provenance, dict)
or provenance.get("schema_version") != PROVENANCE_SCHEMA
or provenance.get("model_name") != MODEL_NAME
or provenance.get("engine_sha256") != EXPECTED_ENGINE_SHA256
or provenance.get("source_model_sha256") != EXPECTED_SOURCE_SHA256
or provenance.get("source_label_sha256") != EXPECTED_LABEL_SHA256
or provenance.get("engine_built_on_target") is not True
):
raise RuntimeError("existing PointPillars provenance is invalid")
def _require_model_ready(url: str) -> None:
request = urllib.request.Request(
f"{url.rstrip('/')}/v2/models/{MODEL_NAME}/ready",
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=30) as response:
if response.status != 200:
raise RuntimeError("PointPillars is not ready")
except OSError as exc:
raise RuntimeError("PointPillars is not ready in canonical Triton") from exc
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(payload: Any) -> bytes:
return json.dumps(
payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _atomic_json(path: Path, payload: dict[str, Any]) -> None:
descriptor, temporary = tempfile.mkstemp(
dir=path.parent,
prefix=f".{path.name}.",
suffix=".tmp",
)
try:
with os.fdopen(descriptor, "wb") as target:
target.write(_canonical_json(payload) + b"\n")
target.flush()
os.fsync(target.fileno())
os.replace(temporary, path)
except BaseException:
with suppress(OSError):
os.unlink(temporary)
raise
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/run_l3_pointpillars_public_baseline.py
+588
View File
@@ -0,0 +1,588 @@
#!/usr/bin/env python3
"""Run the complete KITTI validation baseline through canonical Triton."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
import tempfile
import time
import urllib.request
import zipfile
from contextlib import suppress
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import numpy as np
from k1link.compute.kitti_pointpillars_benchmark import (
PointPillarsFramePrediction,
evaluate_pointpillars_predictions,
read_kitti_validation_truth,
)
from k1link.compute.pointpillars_postprocess import (
POINTPILLARS_EMBEDDED_SCORE_THRESHOLD,
POINTPILLARS_MODEL_POINT_CLOUD_RANGE,
PointPillarsBox,
decode_pointpillars_output,
)
from k1link.datasets.kitti_3d_admission import (
KITTI_3D_RELEASE_ROOT,
KITTI_CALIB_ARCHIVE,
KITTI_LABEL_ARCHIVE,
KITTI_VELODYNE_ARCHIVE,
read_kitti_3d_admission,
read_kitti_standard_splits,
)
RUN_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-run/v1"
FRAME_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-frame/v1"
REPORT_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-report/v1"
MANIFEST_SCHEMA = "missioncore.l3-pointpillars-kitti-transfer-result/v1"
WORKER_PACKAGE_SCHEMA = "missioncore.l3-pointpillars-worker-package/v1"
MODEL_NAME = "pointpillars"
MAXIMUM_POINTS = 204_800
EXPECTED_MODEL_SHA256 = (
"2dcabddc3a365e9608a112d7bbbb7db769a6dddeeaa59aa03611a83113326da1"
)
EXPECTED_LABEL_SHA256 = (
"0adaeb5a374421b61bf83b8fa4522e11abd68461f239a4c72cf5627de913b3da"
)
EXPECTED_ENGINE_SHA256 = (
"12005d972a4632d56342a5da44442b632c1dcc5144fa3c70b162dec334532481"
)
EXPECTED_ONNX_CONTRACT_SHA256 = (
"2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be"
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--model-provenance", type=Path, required=True)
parser.add_argument("--worker-package", type=Path, required=True)
parser.add_argument(
"--output-root",
type=Path,
default=Path(
"/mnt/d/NDC_MISSIONCORE/runtime/experiments/l3/pointpillars-kitti"
),
)
parser.add_argument("--triton-url", default="http://127.0.0.1:8000")
args = parser.parse_args()
dataset_root = args.dataset_root.expanduser().absolute()
admission = read_kitti_3d_admission(dataset_root)
splits = read_kitti_standard_splits(dataset_root)
validation_frame_ids = splits["validation"]
profile_path = args.profile.resolve(strict=True)
provenance_path = args.model_provenance.resolve(strict=True)
profile = _read_json(profile_path)
provenance = _read_json(provenance_path)
_validate_profile_and_provenance(profile, provenance)
worker_package = _read_worker_package(args.worker_package)
_require_triton_ready(args.triton_url)
identity = {
"schema_version": RUN_SCHEMA,
"dataset_source_id": admission["source_id"],
"dataset_release_identity_sha256": admission["release_identity_sha256"],
"validation_frame_count": len(validation_frame_ids),
"validation_frame_ids_sha256": hashlib.sha256(
"\n".join(validation_frame_ids).encode("ascii") + b"\n"
).hexdigest(),
"profile_sha256": _sha256(profile_path),
"model_provenance_sha256": _sha256(provenance_path),
"worker_package_id": worker_package["package_id"],
"worker_package_identity_sha256": worker_package["identity_sha256"],
"engine_sha256": EXPECTED_ENGINE_SHA256,
"producer_sha256": _sha256(Path(__file__)),
"execution": {
"worker_host_id": "worker-006",
"triton_model_name": MODEL_NAME,
"sequential": True,
"parallel_workers": 1,
},
"authority": {
"shadow_only": True,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
run_id = f"l3-pointpillars-kitti-{identity_sha256}"
run_root = args.output_root.expanduser().absolute() / run_id
frames_root = run_root / "frames"
frames_root.mkdir(mode=0o700, parents=True, exist_ok=True)
_write_once(run_root / "identity.json", identity)
archive_root = dataset_root / KITTI_3D_RELEASE_ROOT / "archives"
points_archive = archive_root / KITTI_VELODYNE_ARCHIVE
labels_archive = archive_root / KITTI_LABEL_ARCHIVE
calibrations_archive = archive_root / KITTI_CALIB_ARCHIVE
truth_by_frame = read_kitti_validation_truth(
labels_archive=labels_archive,
calibrations_archive=calibrations_archive,
validation_frame_ids=validation_frame_ids,
)
predictions: list[PointPillarsFramePrediction] = []
try:
with zipfile.ZipFile(points_archive.resolve(strict=True)) as points_zip:
for completed, frame_id in enumerate(validation_frame_ids, start=1):
member = f"training/velodyne/{frame_id}.bin"
point_bytes = points_zip.read(member)
point_sha256 = hashlib.sha256(point_bytes).hexdigest()
frame_path = frames_root / f"{frame_id}.json"
if frame_path.exists():
prediction = _read_frame(
frame_path,
frame_id=frame_id,
point_sha256=point_sha256,
)
else:
prediction = _run_frame(
triton_url=args.triton_url,
frame_id=frame_id,
point_bytes=point_bytes,
)
_atomic_json(
frame_path,
_frame_payload(prediction, point_sha256=point_sha256),
)
predictions.append(prediction)
if completed == 1 or completed % 50 == 0:
print(
json.dumps(
{
"run_id": run_id,
"completed": completed,
"total": len(validation_frame_ids),
"frame_id": frame_id,
},
sort_keys=True,
),
flush=True,
)
except (OSError, KeyError, zipfile.BadZipFile) as exc:
raise RuntimeError("KITTI validation point clouds could not be read") from exc
metrics = evaluate_pointpillars_predictions(
truth_by_frame=truth_by_frame,
predictions=tuple(predictions),
)
report = {
"schema_version": REPORT_SCHEMA,
"run_id": run_id,
"status": "public-cross-domain-transfer-probe-measured",
"dataset_source_id": admission["source_id"],
"dataset_release_identity_sha256": admission["release_identity_sha256"],
"model": {
"name": MODEL_NAME,
"source_model_sha256": EXPECTED_MODEL_SHA256,
"source_label_sha256": EXPECTED_LABEL_SHA256,
"engine_sha256": EXPECTED_ENGINE_SHA256,
},
"metrics": metrics,
"completed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"authority": identity["authority"],
}
_atomic_json(run_root / "report.json", report)
manifest = {
"schema_version": MANIFEST_SCHEMA,
"run_id": run_id,
"identity_sha256": identity_sha256,
"identity": identity,
"status": report["status"],
"artifacts": [
_artifact(run_root / "identity.json", "run-identity"),
_artifact(run_root / "report.json", "benchmark-report"),
],
"frame_result_count": len(predictions),
"frame_results_identity_sha256": _frame_results_identity(frames_root),
"authority": identity["authority"],
}
_atomic_json(run_root / "manifest.json", manifest)
print(
json.dumps(
{
"run_id": run_id,
"status": report["status"],
"frame_count": len(predictions),
"bev_map40": metrics["aggregates"]["bev_map40"],
"3d_map40": metrics["aggregates"]["3d_map40"],
"false_occupied_rate": metrics["aggregates"][
"false_occupied_rate"
],
},
sort_keys=True,
),
flush=True,
)
return 0
def _run_frame(
*,
triton_url: str,
frame_id: str,
point_bytes: bytes,
) -> PointPillarsFramePrediction:
if not point_bytes or len(point_bytes) % 16:
raise RuntimeError("KITTI point frame is not packed XYZI")
native = np.frombuffer(point_bytes, dtype="<f4").reshape(-1, 4)
if not np.isfinite(native).all() or native.shape[0] > MAXIMUM_POINTS:
raise RuntimeError("KITTI point frame violates the model input contract")
points = np.zeros((1, MAXIMUM_POINTS, 4), dtype=np.float32)
points[0, : native.shape[0]] = native
num_points = np.asarray([native.shape[0]], dtype=np.int32)
output_boxes, output_count, elapsed_ms = _infer(
triton_url,
points,
num_points,
)
boxes = decode_pointpillars_output(output_boxes, output_count)
return PointPillarsFramePrediction(
frame_id=frame_id,
boxes=boxes,
inference_ms=elapsed_ms,
)
def _infer(
triton_url: str,
points: np.ndarray,
num_points: np.ndarray,
) -> tuple[np.ndarray, np.ndarray, float]:
points_binary = np.ascontiguousarray(points, dtype=np.float32).tobytes()
count_binary = np.ascontiguousarray(num_points, dtype=np.int32).tobytes()
header = {
"inputs": [
{
"name": "points",
"shape": [1, MAXIMUM_POINTS, 4],
"datatype": "FP32",
"parameters": {"binary_data_size": len(points_binary)},
},
{
"name": "num_points",
"shape": [1],
"datatype": "INT32",
"parameters": {"binary_data_size": len(count_binary)},
},
],
"outputs": [
{"name": "output_boxes", "parameters": {"binary_data": True}},
{"name": "num_boxes", "parameters": {"binary_data": True}},
],
}
encoded_header = _canonical_json(header)
request = urllib.request.Request(
f"{triton_url.rstrip('/')}/v2/models/{MODEL_NAME}/infer",
data=encoded_header + points_binary + count_binary,
headers={
"Content-Type": "application/octet-stream",
"Inference-Header-Content-Length": str(len(encoded_header)),
},
method="POST",
)
started = time.perf_counter()
with urllib.request.urlopen(request, timeout=120) as response:
payload = response.read()
header_length = int(response.headers["Inference-Header-Content-Length"])
elapsed_ms = (time.perf_counter() - started) * 1000.0
response_header = json.loads(payload[:header_length])
outputs = response_header.get("outputs")
if not isinstance(outputs, list) or len(outputs) != 2:
raise RuntimeError("Triton PointPillars output set changed")
offset = header_length
decoded: dict[str, np.ndarray] = {}
for descriptor in outputs:
if not isinstance(descriptor, dict):
raise RuntimeError("Triton PointPillars output descriptor is invalid")
name = descriptor.get("name")
parameters = descriptor.get("parameters")
if not isinstance(name, str) or not isinstance(parameters, dict):
raise RuntimeError("Triton PointPillars output descriptor is invalid")
byte_length = parameters.get("binary_data_size")
if (
isinstance(byte_length, bool)
or not isinstance(byte_length, int)
or byte_length < 1
):
raise RuntimeError("Triton PointPillars binary output size is invalid")
binary = payload[offset : offset + byte_length]
if len(binary) != byte_length:
raise RuntimeError("Triton PointPillars binary output is truncated")
offset += byte_length
if name == "output_boxes" and descriptor.get("datatype") == "FP32":
decoded[name] = np.frombuffer(binary, dtype="<f4").reshape(
1, 393_216, 9
)
elif name == "num_boxes" and descriptor.get("datatype") == "INT32":
decoded[name] = np.frombuffer(binary, dtype="<i4").reshape(1)
else:
raise RuntimeError("Triton PointPillars output contract changed")
if offset != len(payload) or set(decoded) != {"output_boxes", "num_boxes"}:
raise RuntimeError("Triton PointPillars binary payload is invalid")
return decoded["output_boxes"], decoded["num_boxes"], elapsed_ms
def _require_triton_ready(url: str) -> None:
for endpoint in ("/v2/health/ready", f"/v2/models/{MODEL_NAME}/ready"):
request = urllib.request.Request(f"{url.rstrip('/')}{endpoint}", method="GET")
try:
with urllib.request.urlopen(request, timeout=10) as response:
if response.status != 200:
raise RuntimeError("canonical Triton is not ready")
except OSError as exc:
raise RuntimeError("canonical Triton or PointPillars is not ready") from exc
def _validate_profile_and_provenance(
profile: dict[str, Any],
provenance: dict[str, Any],
) -> None:
detector = profile.get("detector")
postprocessing = (
detector.get("postprocessing") if isinstance(detector, dict) else None
)
if (
profile.get("schema_version")
!= "missioncore.l3-pointpillars-benchmark-profile/v1"
or not isinstance(detector, dict)
or not isinstance(postprocessing, dict)
or detector.get("candidate_source_sha256") != EXPECTED_MODEL_SHA256
or detector.get("candidate_label_sha256") != EXPECTED_LABEL_SHA256
or detector.get("triton_model_name") != MODEL_NAME
or detector.get("model_classes")
!= ["Vehicle", "Pedestrian", "Cyclist"]
or detector.get("point_cloud_range")
!= list(POINTPILLARS_MODEL_POINT_CLOUD_RANGE)
or detector.get("training_domain")
!= "proprietary-solid-state-lidar"
or detector.get("training_ground_truth_publicly_reproducible") is not False
or detector.get("onnx_contract_sha256")
!= EXPECTED_ONNX_CONTRACT_SHA256
or postprocessing.get("embedded_score_threshold")
!= POINTPILLARS_EMBEDDED_SCORE_THRESHOLD
or postprocessing.get("embedded_contract_source")
!= "onnx-node-attributes"
):
raise RuntimeError("L3 PointPillars profile is invalid")
if (
provenance.get("model_name") != MODEL_NAME
or provenance.get("source_model_sha256") != EXPECTED_MODEL_SHA256
or provenance.get("source_label_sha256") != EXPECTED_LABEL_SHA256
or provenance.get("engine_sha256") != EXPECTED_ENGINE_SHA256
or provenance.get("engine_built_on_target") is not True
or provenance.get("worker_host_id") != "worker-006"
):
raise RuntimeError("live PointPillars provenance is invalid")
def _read_worker_package(path: Path) -> dict[str, Any]:
package_root = path.expanduser().resolve(strict=True)
runtime_package_root = Path(__file__).resolve(strict=True).parents[1]
if (
not package_root.is_dir()
or package_root != runtime_package_root
or package_root.is_symlink()
):
raise RuntimeError("L3 runner is not executing from its declared worker package")
manifest = _read_json(package_root / "manifest.json")
identity = manifest.get("identity")
identity_sha256 = manifest.get("identity_sha256")
package_id = manifest.get("package_id")
artifacts = manifest.get("artifacts")
if (
manifest.get("schema_version") != WORKER_PACKAGE_SCHEMA
or not isinstance(identity, dict)
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
or package_id != f"l3-pointpillars-worker-package-{identity_sha256}"
or package_root.name != package_id
or not isinstance(artifacts, list)
):
raise RuntimeError("L3 worker package identity is invalid")
expected_paths = identity.get("artifact_paths")
if not isinstance(expected_paths, list) or not expected_paths:
raise RuntimeError("L3 worker package file set is invalid")
expected = set(expected_paths)
actual = {
member.relative_to(package_root).as_posix()
for member in package_root.rglob("*")
if member.is_file()
}
if actual != expected | {"manifest.json"} or len(artifacts) != len(expected):
raise RuntimeError("L3 worker package file set changed")
observed: set[str] = set()
for descriptor in artifacts:
if not isinstance(descriptor, dict):
raise RuntimeError("L3 worker package artifact is invalid")
relative = descriptor.get("path")
artifact = package_root / str(relative)
if (
not isinstance(relative, str)
or relative not in expected
or relative in observed
or Path(relative).is_absolute()
or ".." in Path(relative).parts
or not artifact.is_file()
or artifact.is_symlink()
or descriptor.get("kind") != relative
or descriptor.get("byte_length") != artifact.stat().st_size
or descriptor.get("sha256") != _sha256(artifact)
):
raise RuntimeError("L3 worker package artifact changed")
observed.add(relative)
if observed != expected:
raise RuntimeError("L3 worker package artifact coverage changed")
return manifest
def _frame_payload(
prediction: PointPillarsFramePrediction,
*,
point_sha256: str,
) -> dict[str, Any]:
return {
"schema_version": FRAME_SCHEMA,
"frame_id": prediction.frame_id,
"point_sha256": point_sha256,
"inference_ms": prediction.inference_ms,
"boxes": [
{
"x_m": box.x_m,
"y_m": box.y_m,
"z_m": box.z_m,
"length_m": box.length_m,
"width_m": box.width_m,
"height_m": box.height_m,
"yaw_rad": box.yaw_rad,
"class_id": box.class_id,
"model_class": box.model_class,
"score": box.score,
}
for box in prediction.boxes
],
}
def _read_frame(
path: Path,
*,
frame_id: str,
point_sha256: str,
) -> PointPillarsFramePrediction:
payload = _read_json(path)
boxes = payload.get("boxes")
if (
payload.get("schema_version") != FRAME_SCHEMA
or payload.get("frame_id") != frame_id
or payload.get("point_sha256") != point_sha256
or not isinstance(boxes, list)
):
raise RuntimeError("cached PointPillars frame result is invalid")
try:
decoded = tuple(PointPillarsBox(**box) for box in boxes)
inference_ms = float(payload["inference_ms"])
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError("cached PointPillars frame result is invalid") from exc
if not math.isfinite(inference_ms) or inference_ms <= 0.0:
raise RuntimeError("cached PointPillars frame latency is invalid")
return PointPillarsFramePrediction(
frame_id=frame_id,
boxes=decoded,
inference_ms=inference_ms,
)
def _frame_results_identity(frames_root: Path) -> str:
descriptors = [
{
"name": path.name,
"sha256": _sha256(path),
"byte_length": path.stat().st_size,
}
for path in sorted(frames_root.glob("*.json"))
]
return hashlib.sha256(_canonical_json(descriptors)).hexdigest()
def _artifact(path: Path, role: str) -> dict[str, Any]:
return {
"path": path.name,
"role": role,
"media_type": "application/json",
"sha256": _sha256(path),
"byte_length": path.stat().st_size,
}
def _read_json(path: Path) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError(f"{path.name} is invalid") from exc
if not isinstance(payload, dict):
raise RuntimeError(f"{path.name} is not an object")
return payload
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(payload: Any) -> bytes:
return json.dumps(
payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _write_once(path: Path, payload: dict[str, Any]) -> None:
encoded = _canonical_json(payload) + b"\n"
if path.exists():
if path.read_bytes() != encoded:
raise RuntimeError(f"{path.name} identity changed")
return
_atomic_bytes(path, encoded)
def _atomic_json(path: Path, payload: dict[str, Any]) -> None:
_atomic_bytes(path, _canonical_json(payload) + b"\n")
def _atomic_bytes(path: Path, payload: bytes) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(
dir=path.parent,
prefix=f".{path.name}.",
suffix=".tmp",
)
try:
with os.fdopen(descriptor, "wb") as target:
target.write(payload)
target.flush()
os.fsync(target.fileno())
os.replace(temporary, path)
except BaseException:
with suppress(OSError):
os.unlink(temporary)
raise
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/smoke_l3_pointpillars_live.py
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Execute one schema-only PointPillars smoke frame in canonical Triton."""
from __future__ import annotations
import argparse
import hashlib
import json
import zipfile
from datetime import UTC, datetime
from pathlib import Path
from run_l3_pointpillars_public_baseline import (
EXPECTED_ENGINE_SHA256,
EXPECTED_LABEL_SHA256,
EXPECTED_MODEL_SHA256,
_atomic_json,
_read_json,
_require_triton_ready,
_run_frame,
_validate_profile_and_provenance,
)
from k1link.datasets.kitti_3d_admission import (
KITTI_3D_RELEASE_ROOT,
KITTI_VELODYNE_ARCHIVE,
read_kitti_3d_admission,
read_kitti_standard_splits,
)
SCHEMA = "missioncore.l3-pointpillars-live-smoke/v1"
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--profile", type=Path, required=True)
parser.add_argument("--model-provenance", type=Path, required=True)
parser.add_argument("--frame-id")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--triton-url", default="http://127.0.0.1:8000")
args = parser.parse_args()
dataset_root = args.dataset_root.expanduser().absolute()
admission = read_kitti_3d_admission(dataset_root)
validation = read_kitti_standard_splits(dataset_root)["validation"]
frame_id = args.frame_id or validation[0]
if frame_id not in validation:
raise RuntimeError("smoke frame is not in the admitted validation split")
profile = _read_json(args.profile.resolve(strict=True))
provenance = _read_json(args.model_provenance.resolve(strict=True))
_validate_profile_and_provenance(profile, provenance)
_require_triton_ready(args.triton_url)
points_archive = (
dataset_root
/ KITTI_3D_RELEASE_ROOT
/ "archives"
/ KITTI_VELODYNE_ARCHIVE
)
member = f"training/velodyne/{frame_id}.bin"
try:
with zipfile.ZipFile(points_archive.resolve(strict=True)) as source:
point_bytes = source.read(member)
except (OSError, KeyError, zipfile.BadZipFile) as exc:
raise RuntimeError("smoke point frame could not be read") from exc
prediction = _run_frame(
triton_url=args.triton_url,
frame_id=frame_id,
point_bytes=point_bytes,
)
result = {
"schema_version": SCHEMA,
"status": "engine-schema-executed",
"source_id": admission["source_id"],
"dataset_release_identity_sha256": admission[
"release_identity_sha256"
],
"frame_id": frame_id,
"point_frame_sha256": hashlib.sha256(point_bytes).hexdigest(),
"point_count": len(point_bytes) // 16,
"post_nms_box_count": len(prediction.boxes),
"observed_model_classes": sorted(
{box.model_class for box in prediction.boxes}
),
"inference_ms": prediction.inference_ms,
"model": {
"name": "pointpillars",
"source_model_sha256": EXPECTED_MODEL_SHA256,
"source_label_sha256": EXPECTED_LABEL_SHA256,
"engine_sha256": EXPECTED_ENGINE_SHA256,
},
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"claim_boundary": {
"accuracy_measured": False,
"k1_transfer_evaluated": False,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
_atomic_json(args.output.expanduser().absolute(), result)
print(json.dumps(result, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())
src/k1link/compute/kitti_pointpillars_benchmark.py
+566
View File
@@ -0,0 +1,566 @@
"""KITTI-to-LiDAR truth conversion and bounded PointPillars metrics."""
from __future__ import annotations
import math
import zipfile
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final
import numpy as np
from k1link.compute.pointpillars_postprocess import (
POINTPILLARS_MODEL_POINT_CLOUD_RANGE,
PointPillarsBox,
oriented_3d_iou,
oriented_bev_iou,
)
KITTI_BENCHMARK_CLASSES: Final = ("Car", "Pedestrian", "Cyclist")
MODEL_TO_KITTI_CLASS: Final = {
"Vehicle": "Car",
"Pedestrian": "Pedestrian",
"Cyclist": "Cyclist",
}
KITTI_IOU_THRESHOLDS: Final = {
"Car": 0.7,
"Pedestrian": 0.5,
"Cyclist": 0.5,
}
KITTI_REFERENCE_POINT_CLOUD_RANGE: Final = (
0.0,
-39.68,
-3.0,
69.12,
39.68,
1.0,
)
CROSS_DOMAIN_EVALUATION_RANGE: Final = tuple(
max(
KITTI_REFERENCE_POINT_CLOUD_RANGE[index],
POINTPILLARS_MODEL_POINT_CLOUD_RANGE[index],
)
if index < 3
else min(
KITTI_REFERENCE_POINT_CLOUD_RANGE[index],
POINTPILLARS_MODEL_POINT_CLOUD_RANGE[index],
)
for index in range(6)
)
DISTANCE_BUCKETS_M: Final = ((0.0, 20.0), (20.0, 40.0), (40.0, 70.0))
class KittiPointPillarsBenchmarkError(RuntimeError):
"""KITTI truth or PointPillars predictions violate the frozen contract."""
@dataclass(frozen=True, slots=True)
class KittiLidarTruth:
frame_id: str
benchmark_class: str
x_m: float
y_m: float
z_m: float
length_m: float
width_m: float
height_m: float
yaw_rad: float
@dataclass(frozen=True, slots=True)
class PointPillarsFramePrediction:
frame_id: str
boxes: tuple[PointPillarsBox, ...]
inference_ms: float
@dataclass(frozen=True, slots=True)
class _ScoredPrediction:
frame_id: str
index: int
benchmark_class: str
box: PointPillarsBox
@dataclass(frozen=True, slots=True)
class _MetricEvaluation:
average_precision_40: float
precision: float
recall: float
true_positives: int
false_positives: int
ground_truth_count: int
matched_truth: frozenset[tuple[str, int]]
matched_pairs: tuple[tuple[PointPillarsBox, KittiLidarTruth], ...]
def read_kitti_validation_truth(
*,
labels_archive: Path,
calibrations_archive: Path,
validation_frame_ids: tuple[str, ...],
) -> dict[str, tuple[KittiLidarTruth, ...]]:
"""Read target cuboids and convert camera-bottom centers into LiDAR centers."""
if (
not validation_frame_ids
or len(set(validation_frame_ids)) != len(validation_frame_ids)
or any(len(frame_id) != 6 or not frame_id.isdigit() for frame_id in validation_frame_ids)
):
raise KittiPointPillarsBenchmarkError("KITTI validation frame index is invalid")
try:
with (
zipfile.ZipFile(labels_archive.resolve(strict=True)) as labels_zip,
zipfile.ZipFile(calibrations_archive.resolve(strict=True)) as calib_zip,
):
truth = {
frame_id: _read_frame_truth(labels_zip, calib_zip, frame_id)
for frame_id in validation_frame_ids
}
except (OSError, KeyError, UnicodeDecodeError, zipfile.BadZipFile) as exc:
raise KittiPointPillarsBenchmarkError(
"KITTI validation truth could not be read"
) from exc
if any(
not any(
box.benchmark_class == class_name
for boxes in truth.values()
for box in boxes
)
for class_name in KITTI_BENCHMARK_CLASSES
):
raise KittiPointPillarsBenchmarkError("KITTI validation lacks a target class")
return truth
def evaluate_pointpillars_predictions(
*,
truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]],
predictions: tuple[PointPillarsFramePrediction, ...],
) -> dict[str, Any]:
"""Evaluate the frozen model-to-KITTI mapping without retuning."""
if not truth_by_frame or set(truth_by_frame) != {
prediction.frame_id for prediction in predictions
}:
raise KittiPointPillarsBenchmarkError(
"prediction frames do not equal the admitted validation split"
)
if len(predictions) != len(truth_by_frame):
raise KittiPointPillarsBenchmarkError("prediction frames contain duplicates")
if any(
not math.isfinite(prediction.inference_ms) or prediction.inference_ms <= 0.0
for prediction in predictions
):
raise KittiPointPillarsBenchmarkError("inference latency is invalid")
all_boxes = tuple(
box
for prediction in predictions
for box in prediction.boxes
)
evaluated_boxes = tuple(
box for box in all_boxes if _center_in_evaluation_range(box.x_m, box.y_m, box.z_m)
)
scored = tuple(
_ScoredPrediction(
frame_id=prediction.frame_id,
index=index,
benchmark_class=_benchmark_class(box),
box=box,
)
for prediction in predictions
for index, box in enumerate(prediction.boxes)
if _center_in_evaluation_range(box.x_m, box.y_m, box.z_m)
)
per_class: dict[str, dict[str, Any]] = {}
three_d_evaluations: dict[str, _MetricEvaluation] = {}
for class_name in KITTI_BENCHMARK_CLASSES:
bev = _evaluate_metric(
class_name=class_name,
truth_by_frame=truth_by_frame,
predictions=scored,
iou=oriented_bev_iou,
)
three_d = _evaluate_metric(
class_name=class_name,
truth_by_frame=truth_by_frame,
predictions=scored,
iou=oriented_3d_iou,
)
three_d_evaluations[class_name] = three_d
per_class[class_name] = {
"iou_threshold": KITTI_IOU_THRESHOLDS[class_name],
"bev_ap40": bev.average_precision_40,
"3d_ap40": three_d.average_precision_40,
"precision": three_d.precision,
"recall": three_d.recall,
"true_positives": three_d.true_positives,
"false_positives": three_d.false_positives,
"ground_truth_count": three_d.ground_truth_count,
}
all_three_d_pairs = tuple(
pair
for class_name in KITTI_BENCHMARK_CLASSES
for pair in three_d_evaluations[class_name].matched_pairs
)
errors = _matched_errors(all_three_d_pairs)
total_predictions = sum(
evaluation.true_positives + evaluation.false_positives
for evaluation in three_d_evaluations.values()
)
total_false_positives = sum(
evaluation.false_positives for evaluation in three_d_evaluations.values()
)
latencies = np.asarray(
[prediction.inference_ms for prediction in predictions],
dtype=np.float64,
)
return {
"metric_contract": {
"official_kitti_server_metric": False,
"evaluation_kind": "public-cross-domain-transfer-probe",
"ap_interpolation": "40-point",
"difficulty_filtering": False,
"retuning_on_validation": False,
"model_to_benchmark_class_mapping": MODEL_TO_KITTI_CLASS,
"iou_thresholds": KITTI_IOU_THRESHOLDS,
"model_training_domain": "proprietary-solid-state-lidar",
"model_point_cloud_range": list(
POINTPILLARS_MODEL_POINT_CLOUD_RANGE
),
"dataset_reference_point_cloud_range": list(
KITTI_REFERENCE_POINT_CLOUD_RANGE
),
"shared_evaluation_range": list(CROSS_DOMAIN_EVALUATION_RANGE),
"predictions_outside_shared_range_ignored": True,
},
"frame_count": len(predictions),
"per_class": per_class,
"aggregates": {
"bev_map40": _mean(
[per_class[class_name]["bev_ap40"] for class_name in KITTI_BENCHMARK_CLASSES]
),
"3d_map40": _mean(
[per_class[class_name]["3d_ap40"] for class_name in KITTI_BENCHMARK_CLASSES]
),
"false_occupied_rate": (
total_false_positives / total_predictions
if total_predictions
else 0.0
),
"prediction_volume": {
"model_output_box_count": len(all_boxes),
"evaluated_box_count": len(evaluated_boxes),
"outside_shared_range_count": len(all_boxes)
- len(evaluated_boxes),
},
**errors,
"inference_latency_ms": {
"mean": float(np.mean(latencies)),
"p50": float(np.percentile(latencies, 50)),
"p95": float(np.percentile(latencies, 95)),
"maximum": float(np.max(latencies)),
},
"distance_bucket_recall": _distance_bucket_recall(
truth_by_frame,
three_d_evaluations,
),
},
"claim_boundary": {
"public_cross_domain_transfer_probe": True,
"native_model_accuracy_evaluated": False,
"k1_transfer_evaluated": False,
"navigation_or_safety_accepted": False,
"camera_first_candidate_replaced": False,
},
}
def _read_frame_truth(
labels_zip: zipfile.ZipFile,
calib_zip: zipfile.ZipFile,
frame_id: str,
) -> tuple[KittiLidarTruth, ...]:
calibration = _calibration_transform(
calib_zip.read(f"training/calib/{frame_id}.txt").decode("ascii")
)
labels = labels_zip.read(f"training/label_2/{frame_id}.txt").decode("ascii")
result: list[KittiLidarTruth] = []
for line in labels.splitlines():
fields = line.split()
if not fields or fields[0] not in KITTI_BENCHMARK_CLASSES:
continue
if len(fields) != 15:
raise KittiPointPillarsBenchmarkError("KITTI label row is invalid")
try:
values = np.asarray([float(value) for value in fields[1:]], dtype=np.float64)
except ValueError as exc:
raise KittiPointPillarsBenchmarkError(
"KITTI label row contains invalid numbers"
) from exc
if not np.isfinite(values).all():
raise KittiPointPillarsBenchmarkError(
"KITTI label row contains non-finite numbers"
)
height_m, width_m, length_m = (float(value) for value in values[7:10])
if height_m <= 0.0 or width_m <= 0.0 or length_m <= 0.0:
raise KittiPointPillarsBenchmarkError("KITTI cuboid dimensions are invalid")
camera_bottom_center = np.asarray(
[values[10], values[11], values[12], 1.0],
dtype=np.float64,
)
lidar_bottom_center = calibration @ camera_bottom_center
x_m, y_m, z_m = (float(value) for value in lidar_bottom_center[:3])
z_m += height_m / 2.0
if not _center_in_evaluation_range(x_m, y_m, z_m):
continue
result.append(
KittiLidarTruth(
frame_id=frame_id,
benchmark_class=fields[0],
x_m=x_m,
y_m=y_m,
z_m=z_m,
length_m=length_m,
width_m=width_m,
height_m=height_m,
yaw_rad=_wrap_angle(-(float(values[13]) + math.pi / 2.0)),
)
)
return tuple(result)
def _calibration_transform(payload: str) -> np.ndarray:
values: dict[str, np.ndarray] = {}
for line in payload.splitlines():
if not line.strip():
continue
key, separator, raw = line.partition(":")
if not separator:
raise KittiPointPillarsBenchmarkError("KITTI calibration row is invalid")
try:
row = np.asarray([float(value) for value in raw.split()], dtype=np.float64)
except ValueError as exc:
raise KittiPointPillarsBenchmarkError(
"KITTI calibration contains invalid numbers"
) from exc
if not np.isfinite(row).all():
raise KittiPointPillarsBenchmarkError(
"KITTI calibration contains non-finite numbers"
)
values[key] = row
if "R0_rect" not in values or values["R0_rect"].size != 9:
raise KittiPointPillarsBenchmarkError("KITTI R0_rect is invalid")
if "Tr_velo_to_cam" not in values or values["Tr_velo_to_cam"].size != 12:
raise KittiPointPillarsBenchmarkError("KITTI Tr_velo_to_cam is invalid")
rectification = np.eye(4, dtype=np.float64)
rectification[:3, :3] = values["R0_rect"].reshape(3, 3)
lidar_to_camera = np.eye(4, dtype=np.float64)
lidar_to_camera[:3, :4] = values["Tr_velo_to_cam"].reshape(3, 4)
try:
return np.linalg.inv(rectification @ lidar_to_camera)
except np.linalg.LinAlgError as exc:
raise KittiPointPillarsBenchmarkError(
"KITTI calibration transform is singular"
) from exc
def _center_in_evaluation_range(x_m: float, y_m: float, z_m: float) -> bool:
return (
CROSS_DOMAIN_EVALUATION_RANGE[0]
<= x_m
<= CROSS_DOMAIN_EVALUATION_RANGE[3]
and CROSS_DOMAIN_EVALUATION_RANGE[1]
<= y_m
<= CROSS_DOMAIN_EVALUATION_RANGE[4]
and CROSS_DOMAIN_EVALUATION_RANGE[2]
<= z_m
<= CROSS_DOMAIN_EVALUATION_RANGE[5]
)
def _benchmark_class(box: PointPillarsBox) -> str:
try:
return MODEL_TO_KITTI_CLASS[box.model_class]
except KeyError as exc:
raise KittiPointPillarsBenchmarkError(
"PointPillars model class is not admitted"
) from exc
def _truth_as_box(truth: KittiLidarTruth) -> PointPillarsBox:
return PointPillarsBox(
x_m=truth.x_m,
y_m=truth.y_m,
z_m=truth.z_m,
length_m=truth.length_m,
width_m=truth.width_m,
height_m=truth.height_m,
yaw_rad=truth.yaw_rad,
class_id=-1,
model_class=truth.benchmark_class,
score=1.0,
)
def _evaluate_metric(
*,
class_name: str,
truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]],
predictions: tuple[_ScoredPrediction, ...],
iou: Callable[[PointPillarsBox, PointPillarsBox], float],
) -> _MetricEvaluation:
truths = {
frame_id: tuple(
box for box in boxes if box.benchmark_class == class_name
)
for frame_id, boxes in truth_by_frame.items()
}
ground_truth_count = sum(len(boxes) for boxes in truths.values())
ordered = sorted(
(
prediction
for prediction in predictions
if prediction.benchmark_class == class_name
),
key=lambda prediction: (
-prediction.box.score,
prediction.frame_id,
prediction.index,
),
)
matched: set[tuple[str, int]] = set()
matched_pairs: list[tuple[PointPillarsBox, KittiLidarTruth]] = []
true_positive_flags: list[int] = []
false_positive_flags: list[int] = []
threshold = KITTI_IOU_THRESHOLDS[class_name]
for prediction in ordered:
candidates = truths[prediction.frame_id]
best_index = -1
best_iou = -1.0
for truth_index, truth in enumerate(candidates):
if (prediction.frame_id, truth_index) in matched:
continue
overlap = iou(prediction.box, _truth_as_box(truth))
if overlap > best_iou:
best_iou = overlap
best_index = truth_index
if best_index >= 0 and best_iou >= threshold:
matched.add((prediction.frame_id, best_index))
matched_pairs.append((prediction.box, candidates[best_index]))
true_positive_flags.append(1)
false_positive_flags.append(0)
else:
true_positive_flags.append(0)
false_positive_flags.append(1)
cumulative_true = np.cumsum(true_positive_flags, dtype=np.float64)
cumulative_false = np.cumsum(false_positive_flags, dtype=np.float64)
precision = np.divide(
cumulative_true,
np.maximum(cumulative_true + cumulative_false, 1.0),
)
recall = cumulative_true / max(float(ground_truth_count), 1.0)
average_precision = _ap40(precision, recall)
return _MetricEvaluation(
average_precision_40=average_precision,
precision=float(precision[-1]) if precision.size else 0.0,
recall=float(recall[-1]) if recall.size else 0.0,
true_positives=int(cumulative_true[-1]) if cumulative_true.size else 0,
false_positives=int(cumulative_false[-1]) if cumulative_false.size else 0,
ground_truth_count=ground_truth_count,
matched_truth=frozenset(matched),
matched_pairs=tuple(matched_pairs),
)
def _ap40(precision: np.ndarray, recall: np.ndarray) -> float:
if precision.size == 0:
return 0.0
interpolated = [
float(np.max(precision[recall >= threshold]))
if np.any(recall >= threshold)
else 0.0
for threshold in np.arange(40, dtype=np.float64) / 40.0
]
return _mean(interpolated)
def _matched_errors(
pairs: tuple[tuple[PointPillarsBox, KittiLidarTruth], ...],
) -> dict[str, Any]:
center_errors = [
math.dist(
(prediction.x_m, prediction.y_m, prediction.z_m),
(truth.x_m, truth.y_m, truth.z_m),
)
for prediction, truth in pairs
]
range_errors = [
abs(
math.hypot(prediction.x_m, prediction.y_m)
- math.hypot(truth.x_m, truth.y_m)
)
for prediction, truth in pairs
]
yaw_errors = [
abs(_wrap_angle(prediction.yaw_rad - truth.yaw_rad))
for prediction, truth in pairs
]
return {
"center_error_m": _error_summary(center_errors),
"range_error_m": _error_summary(range_errors),
"yaw_error_rad": _error_summary(yaw_errors),
}
def _error_summary(values: list[float]) -> dict[str, float | int | None]:
if not values:
return {"count": 0, "mean": None, "p95": None, "maximum": None}
array = np.asarray(values, dtype=np.float64)
return {
"count": len(values),
"mean": float(np.mean(array)),
"p95": float(np.percentile(array, 95)),
"maximum": float(np.max(array)),
}
def _distance_bucket_recall(
truth_by_frame: dict[str, tuple[KittiLidarTruth, ...]],
evaluations: dict[str, _MetricEvaluation],
) -> dict[str, dict[str, float | int]]:
result: dict[str, dict[str, float | int]] = {}
for minimum, maximum in DISTANCE_BUCKETS_M:
total = 0
matched = 0
for frame_id, truths in truth_by_frame.items():
by_class_index = {class_name: 0 for class_name in KITTI_BENCHMARK_CLASSES}
for truth in truths:
index = by_class_index[truth.benchmark_class]
by_class_index[truth.benchmark_class] += 1
distance = math.hypot(truth.x_m, truth.y_m)
if not minimum <= distance < maximum:
continue
total += 1
if (frame_id, index) in evaluations[
truth.benchmark_class
].matched_truth:
matched += 1
result[f"{int(minimum)}-{int(maximum)}m"] = {
"ground_truth_count": total,
"matched_count": matched,
"recall": matched / total if total else 0.0,
}
return result
def _wrap_angle(value: float) -> float:
return (value + math.pi) % (2.0 * math.pi) - math.pi
def _mean(values: list[float]) -> float:
return sum(values) / len(values) if values else 0.0
src/k1link/compute/l3_pointpillars_admission.py
+828
View File
@@ -0,0 +1,828 @@
"""Fail-closed admission for the L3 NVIDIA PointPillars benchmark.
The admission deliberately separates three different claims:
* a detector-quality baseline needs independent oriented 3D box truth;
* K1 transfer can measure runtime and representation stability only after that
baseline has been accepted;
* point-wise semantic or instance labels are valuable evidence, but are not a
substitute for 3D cuboid truth.
The result is immutable and content addressed. It grants no command,
navigation, or safety authority and cannot start a worker or install a model.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final
from k1link.artifacts import utc_now_iso
from k1link.compute.pointpillars_postprocess import (
POINTPILLARS_EMBEDDED_SCORE_THRESHOLD,
POINTPILLARS_MODEL_POINT_CLOUD_RANGE,
)
L3_PROFILE_SCHEMA: Final = "missioncore.l3-pointpillars-benchmark-profile/v1"
L3_DATASET_INVENTORY_SCHEMA: Final = "missioncore.l3-lidar-dataset-inventory/v1"
L3_WORKER_INVENTORY_SCHEMA: Final = "missioncore.l3-worker-inventory/v1"
L3_ADMISSION_SCHEMA: Final = "missioncore.l3-pointpillars-admission/v1"
L3_REPORT_SCHEMA: Final = "missioncore.l3-pointpillars-admission-report/v1"
L3_REPORT_NAME: Final = "admission-report.json"
L3_MANIFEST_NAME: Final = "manifest.json"
_RESULT_ID = re.compile(r"^l3-pointpillars-admission-[a-f0-9]{64}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,191}$")
_BOX_ANNOTATION = "oriented-3d-boxes"
_ONNX_CONTRACT_SHA256 = (
"2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be"
)
_POINT_FIELDS = ("x", "y", "z", "intensity")
_MODEL_CLASSES = ("Vehicle", "Pedestrian", "Cyclist")
_BENCHMARK_CLASSES = ("Car", "Pedestrian", "Cyclist")
_BENCHMARK_CLASS_MAPPING = {
"Vehicle": "Car",
"Pedestrian": "Pedestrian",
"Cyclist": "Cyclist",
}
_POSTPROCESSING_CONTRACT = {
"reference_repository": "https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes",
"reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8",
"output_row_fields": [
"x",
"y",
"z",
"length",
"width",
"height",
"yaw",
"class_id",
"score",
],
"class_agnostic_nms": True,
"nms_iou_threshold": 0.01,
"pre_nms_top_n": 4096,
"embedded_score_threshold": POINTPILLARS_EMBEDDED_SCORE_THRESHOLD,
"embedded_contract_source": "onnx-node-attributes",
}
_BENCHMARK_METRIC_CONTRACT = {
"official_kitti_server_metric": False,
"evaluation_kind": "public-cross-domain-transfer-probe",
"ap_interpolation": "40-point",
"difficulty_filtering": False,
"predictions_outside_shared_range_ignored": True,
"iou_thresholds": {
"Car": 0.7,
"Pedestrian": 0.5,
"Cyclist": 0.5,
},
"distance_buckets_m": [[0, 20], [20, 40], [40, 70]],
}
_POINT_CLOUD_RANGE = POINTPILLARS_MODEL_POINT_CLOUD_RANGE
_PUBLIC_TRANSFER_METRICS = (
"bev-map",
"3d-map",
"center-error-m",
"range-error-m",
"yaw-error-rad",
"distance-bucket-recall",
"false-occupied-rate",
"end-to-end-latency-ms",
)
_TRANSFER_METRICS = (
"input-admission-rate",
"output-schema-valid-rate",
"deterministic-replay-rate",
"end-to-end-latency-ms",
"queue-wait-ms",
"drop-rate",
)
class L3PointPillarsAdmissionError(RuntimeError):
"""An L3 profile, inventory, or immutable result is invalid."""
@dataclass(frozen=True, slots=True)
class L3PointPillarsAdmission:
result_root: Path
result_id: str
manifest: dict[str, Any]
report: dict[str, Any]
@property
def public_transfer_probe_authorized(self) -> bool:
decision = _object(self.report.get("decision"), "L3 decision")
return decision.get("public_cross_domain_probe_authorized") is True
def build_l3_pointpillars_admission(
*,
profile_path: Path,
dataset_inventory_path: Path,
worker_inventory_path: Path,
output_root: Path,
) -> L3PointPillarsAdmission:
"""Build or reopen one immutable L3 benchmark admission."""
profile_path = profile_path.resolve(strict=True)
dataset_inventory_path = dataset_inventory_path.resolve(strict=True)
worker_inventory_path = worker_inventory_path.resolve(strict=True)
profile = _read_json(profile_path)
dataset_inventory = _read_json(dataset_inventory_path)
worker_inventory = _read_json(worker_inventory_path)
_validate_profile(profile)
_validate_dataset_inventory(dataset_inventory)
_validate_worker_inventory(worker_inventory)
identity = {
"schema_version": L3_ADMISSION_SCHEMA,
"profile": profile,
"profile_sha256": _sha256(profile_path),
"dataset_inventory_sha256": _sha256(dataset_inventory_path),
"worker_inventory_sha256": _sha256(worker_inventory_path),
"producer_sha256": _sha256(Path(__file__)),
"authority": _authority(),
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"l3-pointpillars-admission-{identity_sha256}"
destination = output_root.expanduser().absolute()
destination.mkdir(mode=0o700, parents=True, exist_ok=True)
result_root = destination / result_id
if result_root.exists():
return read_l3_pointpillars_admission(result_root)
report = _build_report(
result_id=result_id,
profile=profile,
dataset_inventory=dataset_inventory,
worker_inventory=worker_inventory,
)
staging = destination / f".{result_id}.{os.getpid()}.incomplete"
staging.mkdir(mode=0o700, exist_ok=False)
try:
_write_json(staging / L3_REPORT_NAME, report)
report_artifact = _artifact(
staging / L3_REPORT_NAME,
"l3-pointpillars-admission-report",
)
manifest = {
"schema_version": L3_ADMISSION_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"status": report["status"],
"artifacts": [report_artifact],
"created_at_utc": utc_now_iso(),
"authority": _authority(),
}
_write_json(staging / L3_MANIFEST_NAME, manifest)
os.replace(staging, result_root)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return read_l3_pointpillars_admission(result_root)
def read_l3_pointpillars_admission(root: Path) -> L3PointPillarsAdmission:
"""Read and fully validate an immutable L3 admission result."""
resolved = root.expanduser().resolve(strict=True)
if not resolved.is_dir() or _RESULT_ID.fullmatch(resolved.name) is None:
raise L3PointPillarsAdmissionError("L3 result root is invalid")
manifest = _read_json(resolved / L3_MANIFEST_NAME)
report = _read_json(resolved / L3_REPORT_NAME)
if (
manifest.get("schema_version") != L3_ADMISSION_SCHEMA
or manifest.get("result_id") != resolved.name
or report.get("schema_version") != L3_REPORT_SCHEMA
or report.get("result_id") != resolved.name
or manifest.get("status") != report.get("status")
or manifest.get("authority") != _authority()
or report.get("authority") != _authority()
):
raise L3PointPillarsAdmissionError("L3 manifest and report are inconsistent")
identity = _object(manifest.get("identity"), "L3 identity")
expected_identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
if (
manifest.get("identity_sha256") != expected_identity_sha256
or resolved.name != f"l3-pointpillars-admission-{expected_identity_sha256}"
):
raise L3PointPillarsAdmissionError("L3 result identity is invalid")
artifacts = _array(manifest.get("artifacts"), "L3 artifacts")
if len(artifacts) != 1:
raise L3PointPillarsAdmissionError("L3 result has an invalid artifact set")
artifact = _object(artifacts[0], "L3 report artifact")
report_path = resolved / L3_REPORT_NAME
if (
artifact.get("role") != "l3-pointpillars-admission-report"
or artifact.get("path") != L3_REPORT_NAME
or artifact.get("media_type") != "application/json"
or artifact.get("sha256") != _sha256(report_path)
or artifact.get("byte_length") != report_path.stat().st_size
):
raise L3PointPillarsAdmissionError("L3 report artifact is invalid")
_validate_report(report)
return L3PointPillarsAdmission(
result_root=resolved,
result_id=resolved.name,
manifest=manifest,
report=report,
)
def _build_report(
*,
result_id: str,
profile: dict[str, Any],
dataset_inventory: dict[str, Any],
worker_inventory: dict[str, Any],
) -> dict[str, Any]:
detector = _object(profile.get("detector"), "L3 detector")
runtime_policy = _object(profile.get("runtime_policy"), "L3 runtime policy")
public_probe = _object(
profile.get("public_cross_domain_probe"),
"L3 public cross-domain probe",
)
required_model = _required_string(detector, "triton_model_name")
candidate_frozen = detector.get("candidate_frozen") is True
required_version = detector.get("candidate_model_version")
required_source_sha256 = detector.get("candidate_source_sha256")
required_label_sha256 = detector.get("candidate_label_sha256")
triton = _object(worker_inventory.get("triton"), "L3 Triton inventory")
installed_models = [
_object(item, "L3 worker model")
for item in _array(triton.get("models"), "L3 worker models")
]
matching_models = [
model
for model in installed_models
if candidate_frozen
and model.get("name") == required_model
and model.get("upstream_version") == required_version
and model.get("source_model_sha256") == required_source_sha256
and model.get("source_label_sha256") == required_label_sha256
]
model_ready = any(_model_ready(model) for model in matching_models)
staged_models = [
_object(item, "L3 staged worker model")
for item in _array(
worker_inventory.get("staged_models"),
"L3 staged worker models",
)
]
matching_staged_models = [
model
for model in staged_models
if candidate_frozen
and model.get("name") == required_model
and model.get("upstream_version") == required_version
and model.get("source_model_sha256") == required_source_sha256
and model.get("source_label_sha256") == required_label_sha256
]
staged_model_ready = any(_model_ready(model) for model in matching_staged_models)
datasets = [
_object(item, "L3 dataset")
for item in _array(dataset_inventory.get("datasets"), "L3 datasets")
]
dataset_findings = [_dataset_finding(dataset) for dataset in datasets]
eligible_datasets = [
dataset
for dataset in datasets
if _dataset_supports_public_probe(dataset, public_probe)
]
runtime_checks = {
"canonical_triton_healthy": triton.get("healthy") is True,
"canonical_triton_image_pinned": (
triton.get("image") == runtime_policy.get("triton_image")
and triton.get("image_digest") == runtime_policy.get("triton_image_digest")
),
"explicit_model_control": triton.get("model_control_mode") == "explicit",
"strict_readiness": triton.get("strict_readiness") is True,
"model_repository_read_only": triton.get("model_repository_read_only") is True,
"second_serving_stack_absent": (
worker_inventory.get("serving_stack_count") == 1
),
}
blockers: list[str] = []
if not all(runtime_checks.values()):
blockers.append("canonical-triton-runtime-policy-not-satisfied")
if not eligible_datasets:
blockers.append("public-oriented-3d-box-truth-not-admitted")
if not candidate_frozen:
blockers.append("pointpillars-compatible-candidate-not-frozen")
elif not matching_models:
blockers.append(
"pointpillars-model-not-installed-live"
if staged_model_ready
else "pointpillars-model-artifact-not-installed"
)
elif not model_ready:
blockers.append("pointpillars-target-engine-or-provenance-not-verified")
public_probe_authorized = not blockers
status = (
"ready-for-public-cross-domain-probe"
if public_probe_authorized
else "blocked-foundation-assets"
)
next_gate = (
"run-public-cross-domain-pointpillars-probe"
if public_probe_authorized
else _next_gate(blockers)
)
return {
"schema_version": L3_REPORT_SCHEMA,
"result_id": result_id,
"status": status,
"profile_id": profile["profile_id"],
"observations": {
"dataset_inventory_observed_at_utc": dataset_inventory["observed_at_utc"],
"worker_inventory_observed_at_utc": worker_inventory["observed_at_utc"],
"worker_host_id": worker_inventory["host_id"],
},
"detector": {
"family": detector["family"],
"upstream_model_id": detector["upstream_model_id"],
"candidate_frozen": candidate_frozen,
"candidate_model_version": required_version,
"candidate_source_sha256": required_source_sha256,
"candidate_label_sha256": required_label_sha256,
"triton_model_name": required_model,
"required_input_fields": list(_POINT_FIELDS),
"maximum_points": detector["maximum_points"],
"point_cloud_range": list(_POINT_CLOUD_RANGE),
"training_domain": detector["training_domain"],
"training_ground_truth_publicly_reproducible": False,
"model_classes": list(_MODEL_CLASSES),
"benchmark_classes": list(_BENCHMARK_CLASSES),
"benchmark_class_mapping": _BENCHMARK_CLASS_MAPPING,
"benchmark_metric_contract": _BENCHMARK_METRIC_CONTRACT,
"postprocessing": _POSTPROCESSING_CONTRACT,
"installed_matching_model_count": len(matching_models),
"model_ready": model_ready,
"staged_matching_model_count": len(matching_staged_models),
"staged_target_engine_ready": staged_model_ready,
},
"runtime_checks": runtime_checks,
"dataset_findings": dataset_findings,
"eligible_public_probe_dataset_ids": [
_required_string(dataset, "dataset_id") for dataset in eligible_datasets
],
"blocker_codes": blockers,
"decision": {
"public_cross_domain_probe_authorized": public_probe_authorized,
"native_model_accuracy_claim_authorized": False,
"k1_transfer_stability_authorized": False,
"k1_transfer_quality_claim_authorized": False,
"semantic_point_labels_substitute_for_3d_boxes": False,
"fine_tuning_allowed": False,
"second_serving_stack_allowed": False,
"lab_publication_allowed": False,
"centerpoint_comparison_allowed": False,
},
"claim_boundaries": {
"public_cross_domain_probe_metrics": list(_PUBLIC_TRANSFER_METRICS),
"k1_transfer_stability_metrics": list(_TRANSFER_METRICS),
"public_probe_is_native_model_accuracy": False,
"k1_accuracy_requires_independent_truth": True,
"absence_of_detection_means_free_space": False,
"point_instance_clusters_are_3d_box_truth": False,
},
"next_gate": next_gate,
"authority": _authority(),
}
def _dataset_finding(dataset: dict[str, Any]) -> dict[str, Any]:
annotations = set(_string_array(dataset.get("annotations"), "dataset annotations"))
installed = dataset.get("installed") is True
has_box_truth = _BOX_ANNOTATION in annotations
return {
"dataset_id": _required_string(dataset, "dataset_id"),
"installed": installed,
"point_fields": _string_array(dataset.get("point_fields"), "dataset point fields"),
"annotations": sorted(annotations),
"independent_ground_truth": dataset.get("independent_ground_truth") is True,
"oriented_3d_box_accuracy_eligible": (
installed
and has_box_truth
and dataset.get("independent_ground_truth") is True
and set(_POINT_FIELDS).issubset(
set(_string_array(dataset.get("point_fields"), "dataset point fields"))
)
),
"semantic_or_instance_labels_are_not_boxes": (
not has_box_truth
and bool(
annotations.intersection(
{"point-semantic-labels", "point-instance-labels"}
)
)
),
}
def _dataset_supports_public_probe(
dataset: dict[str, Any],
public_probe: dict[str, Any],
) -> bool:
splits = set(_string_array(dataset.get("splits"), "dataset splits"))
point_fields = set(
_string_array(dataset.get("point_fields"), "dataset point fields")
)
annotations = set(
_string_array(dataset.get("annotations"), "dataset annotations")
)
required_split = _required_string(public_probe, "required_split")
return (
dataset.get("installed") is True
and dataset.get("independent_ground_truth") is True
and required_split in splits
and set(_POINT_FIELDS).issubset(point_fields)
and _BOX_ANNOTATION in annotations
)
def _model_ready(model: dict[str, Any]) -> bool:
smoke = model.get("representation_smoke")
return (
_valid_sha256(model.get("artifact_sha256"))
and model.get("backend") == "tensorrt"
and model.get("precision") == "strongly-typed"
and model.get("engine_built_on_target") is True
and model.get("provenance_verified") is True
and model.get("source_format") == "onnx"
and model.get("input_fields") == list(_POINT_FIELDS)
and model.get("maximum_points") == 204_800
and model.get("point_cloud_range") == list(_POINT_CLOUD_RANGE)
and model.get("model_classes") == list(_MODEL_CLASSES)
and _valid_sha256(model.get("source_label_sha256"))
and model.get("outputs")
== [
{
"name": "output_boxes",
"dtype": "FP32",
"shape": [1, 393_216, 9],
},
{"name": "num_boxes", "dtype": "INT32", "shape": [1]},
]
and isinstance(smoke, dict)
and smoke.get("status") == "engine-executed"
and _valid_sha256(smoke.get("input_artifact_sha256"))
and smoke.get("input_point_count") == 169_883
and isinstance(smoke.get("single_query_gpu_compute_ms"), float)
and 0.0 < smoke["single_query_gpu_compute_ms"] < 1000.0
and smoke.get("accuracy_evaluated") is False
and smoke.get("navigation_or_safety_accepted") is False
)
def _next_gate(blockers: list[str]) -> str:
missing_dataset = "public-oriented-3d-box-truth-not-admitted" in blockers
missing_candidate = "pointpillars-compatible-candidate-not-frozen" in blockers
missing_model = "pointpillars-model-artifact-not-installed" in blockers
staged_model = "pointpillars-model-not-installed-live" in blockers
if missing_dataset and missing_candidate:
return "admit-public-3d-box-split-and-freeze-compatible-pointpillars-candidate"
if missing_dataset and missing_model:
return "admit-public-3d-box-split-and-build-target-pointpillars-engine"
if missing_dataset and staged_model:
return "admit-public-3d-box-split-then-install-staged-pointpillars-model"
if missing_dataset:
return "admit-public-oriented-3d-box-validation-split"
if missing_model:
return "build-and-admit-target-pointpillars-engine"
if staged_model:
return "install-staged-pointpillars-model-in-canonical-triton"
if missing_candidate:
return "freeze-compatible-pointpillars-onnx-candidate"
if "pointpillars-target-engine-or-provenance-not-verified" in blockers:
return "verify-target-engine-and-model-provenance"
return "repair-canonical-triton-runtime-policy"
def _validate_profile(profile: dict[str, Any]) -> None:
if profile.get("schema_version") != L3_PROFILE_SCHEMA:
raise L3PointPillarsAdmissionError("L3 profile schema is invalid")
_safe_identifier(_required_string(profile, "profile_id"), "L3 profile id")
detector = _object(profile.get("detector"), "L3 detector")
if (
detector.get("family") != "nvidia-tao-pointpillars"
or detector.get("upstream_model_id") != "nvidia/tao/pointpillarnet"
or detector.get("triton_model_name") != "pointpillars"
or detector.get("required_source_format") != "onnx"
or detector.get("input_representation") != "native-sensor-scan"
or detector.get("input_coordinate_frame") != "sensor/lidar"
or detector.get("input_fields") != list(_POINT_FIELDS)
or detector.get("batch_size") != 1
or detector.get("maximum_points") != 204_800
or detector.get("point_cloud_range") != list(_POINT_CLOUD_RANGE)
or detector.get("training_domain") != "proprietary-solid-state-lidar"
or detector.get("training_ground_truth_publicly_reproducible") is not False
or detector.get("onnx_contract_sha256") != _ONNX_CONTRACT_SHA256
or detector.get("model_classes") != list(_MODEL_CLASSES)
):
raise L3PointPillarsAdmissionError("L3 detector contract is invalid")
candidate_frozen = detector.get("candidate_frozen")
candidate_version = detector.get("candidate_model_version")
candidate_sha256 = detector.get("candidate_source_sha256")
candidate_label_sha256 = detector.get("candidate_label_sha256")
if (
not isinstance(candidate_frozen, bool)
or (
candidate_frozen
and (
not isinstance(candidate_version, str)
or not candidate_version
or not _valid_sha256(candidate_sha256)
or not _valid_sha256(candidate_label_sha256)
)
)
or (
not candidate_frozen
and (
candidate_version is not None
or candidate_sha256 is not None
or candidate_label_sha256 is not None
)
)
):
raise L3PointPillarsAdmissionError("L3 detector candidate freeze is invalid")
if detector.get("postprocessing") != _POSTPROCESSING_CONTRACT:
raise L3PointPillarsAdmissionError("L3 detector postprocessing is invalid")
runtime = _object(profile.get("runtime_policy"), "L3 runtime policy")
if (
runtime.get("existing_triton_only") is not True
or runtime.get("second_serving_stack_allowed") is not False
or runtime.get("engine_built_on_target_required") is not True
or runtime.get("precision") != "strongly-typed"
or runtime.get("triton_image") != "nvcr.io/nvidia/tritonserver:26.06-py3"
or not _valid_sha256(runtime.get("triton_image_digest"))
):
raise L3PointPillarsAdmissionError("L3 runtime policy is invalid")
public_probe = _object(
profile.get("public_cross_domain_probe"),
"L3 public cross-domain probe",
)
if (
public_probe.get("required_split") != "validation"
or public_probe.get("required_ground_truth") != _BOX_ANNOTATION
or public_probe.get("independent_ground_truth_required") is not True
or public_probe.get("benchmark_classes") != list(_BENCHMARK_CLASSES)
or public_probe.get("model_to_benchmark_class_mapping")
!= _BENCHMARK_CLASS_MAPPING
or public_probe.get("metric_contract") != _BENCHMARK_METRIC_CONTRACT
or public_probe.get("metrics") != list(_PUBLIC_TRANSFER_METRICS)
or public_probe.get("retuning_allowed") is not False
or public_probe.get("native_accuracy_claim_allowed") is not False
):
raise L3PointPillarsAdmissionError(
"L3 public cross-domain probe is invalid"
)
transfer = _object(profile.get("k1_transfer_stability"), "L3 transfer gate")
if (
transfer.get("requires_completed_public_cross_domain_probe") is not True
or transfer.get("metrics") != list(_TRANSFER_METRICS)
or transfer.get("accuracy_claim_allowed") is not False
or transfer.get("retuning_allowed") is not False
):
raise L3PointPillarsAdmissionError("L3 K1 transfer gate is invalid")
if profile.get("authority") != _authority():
raise L3PointPillarsAdmissionError("L3 profile authority is invalid")
def _validate_dataset_inventory(inventory: dict[str, Any]) -> None:
if inventory.get("schema_version") != L3_DATASET_INVENTORY_SCHEMA:
raise L3PointPillarsAdmissionError("L3 dataset inventory schema is invalid")
_utc(_required_string(inventory, "observed_at_utc"), "dataset observation time")
datasets = _array(inventory.get("datasets"), "L3 datasets")
if not datasets:
raise L3PointPillarsAdmissionError("L3 dataset inventory is empty")
seen: set[str] = set()
for raw_dataset in datasets:
dataset = _object(raw_dataset, "L3 dataset")
dataset_id = _safe_identifier(
_required_string(dataset, "dataset_id"),
"L3 dataset id",
)
if dataset_id in seen:
raise L3PointPillarsAdmissionError("L3 dataset inventory has duplicates")
seen.add(dataset_id)
installed = dataset.get("installed")
if not isinstance(installed, bool):
raise L3PointPillarsAdmissionError("L3 dataset installed flag is invalid")
release_identity = dataset.get("release_identity_sha256")
if installed and not _valid_sha256(release_identity):
raise L3PointPillarsAdmissionError(
"installed L3 dataset lacks a release identity"
)
if not installed and release_identity is not None:
raise L3PointPillarsAdmissionError(
"absent L3 dataset cannot claim a release identity"
)
point_fields = _string_array(
dataset.get("point_fields"),
"dataset point fields",
)
if any(field not in _POINT_FIELDS for field in point_fields):
raise L3PointPillarsAdmissionError("L3 dataset point field is unknown")
_string_array(dataset.get("annotations"), "dataset annotations")
_string_array(dataset.get("splits"), "dataset splits")
if not isinstance(dataset.get("independent_ground_truth"), bool):
raise L3PointPillarsAdmissionError(
"L3 dataset truth independence flag is invalid"
)
_required_string(dataset, "license")
def _validate_worker_inventory(inventory: dict[str, Any]) -> None:
if inventory.get("schema_version") != L3_WORKER_INVENTORY_SCHEMA:
raise L3PointPillarsAdmissionError("L3 worker inventory schema is invalid")
_safe_identifier(_required_string(inventory, "host_id"), "L3 worker host id")
_utc(_required_string(inventory, "observed_at_utc"), "worker observation time")
serving_stack_count = inventory.get("serving_stack_count")
if (
isinstance(serving_stack_count, bool)
or not isinstance(serving_stack_count, int)
or serving_stack_count < 0
):
raise L3PointPillarsAdmissionError("L3 serving stack count is invalid")
triton = _object(inventory.get("triton"), "L3 Triton inventory")
_required_string(triton, "container_name")
_required_string(triton, "image")
if not _valid_sha256(triton.get("image_digest")):
raise L3PointPillarsAdmissionError("L3 Triton image digest is invalid")
for key in ("healthy", "strict_readiness", "model_repository_read_only"):
if not isinstance(triton.get(key), bool):
raise L3PointPillarsAdmissionError(f"L3 Triton {key} flag is invalid")
_required_string(triton, "model_control_mode")
models = _array(triton.get("models"), "L3 worker models")
seen: set[str] = set()
for raw_model in models:
model = _object(raw_model, "L3 worker model")
name = _safe_identifier(_required_string(model, "name"), "L3 worker model name")
if name in seen:
raise L3PointPillarsAdmissionError("L3 worker inventory has duplicate models")
seen.add(name)
if not _valid_sha256(model.get("artifact_sha256")):
raise L3PointPillarsAdmissionError("L3 worker model identity is invalid")
_required_string(model, "backend")
staged_models = _array(inventory.get("staged_models"), "L3 staged worker models")
staged_seen: set[str] = set()
for raw_model in staged_models:
model = _object(raw_model, "L3 staged worker model")
name = _safe_identifier(
_required_string(model, "name"),
"L3 staged worker model name",
)
if name in staged_seen:
raise L3PointPillarsAdmissionError(
"L3 worker inventory has duplicate staged models"
)
staged_seen.add(name)
if not _valid_sha256(model.get("artifact_sha256")):
raise L3PointPillarsAdmissionError(
"L3 staged worker model identity is invalid"
)
_required_string(model, "backend")
gpu = _object(inventory.get("gpu"), "L3 GPU inventory")
_required_string(gpu, "name")
_required_string(gpu, "driver_version")
memory_mib = gpu.get("memory_total_mib")
if (
isinstance(memory_mib, bool)
or not isinstance(memory_mib, int)
or memory_mib <= 0
):
raise L3PointPillarsAdmissionError("L3 GPU memory declaration is invalid")
def _validate_report(report: dict[str, Any]) -> None:
if report.get("schema_version") != L3_REPORT_SCHEMA:
raise L3PointPillarsAdmissionError("L3 report schema is invalid")
decision = _object(report.get("decision"), "L3 decision")
public_probe = decision.get("public_cross_domain_probe_authorized")
transfer = decision.get("k1_transfer_stability_authorized")
blockers = _string_array(report.get("blocker_codes"), "L3 blockers")
if (
not isinstance(public_probe, bool)
or not isinstance(transfer, bool)
or transfer is not False
or decision.get("native_model_accuracy_claim_authorized") is not False
or decision.get("k1_transfer_quality_claim_authorized") is not False
or decision.get("semantic_point_labels_substitute_for_3d_boxes") is not False
or decision.get("fine_tuning_allowed") is not False
or decision.get("second_serving_stack_allowed") is not False
or decision.get("lab_publication_allowed") is not False
or decision.get("centerpoint_comparison_allowed") is not False
or (public_probe and blockers)
or (not public_probe and not blockers)
or report.get("authority") != _authority()
):
raise L3PointPillarsAdmissionError("L3 report decision is invalid")
def _authority() -> dict[str, bool]:
return {
"shadow_only": True,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
def _artifact(path: Path, role: str) -> dict[str, object]:
return {
"role": role,
"path": path.name,
"media_type": "application/json",
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise L3PointPillarsAdmissionError(
f"cannot read L3 JSON: {path.name}"
) from exc
return _object(value, f"L3 JSON {path.name}")
def _write_json(path: Path, value: object) -> None:
path.write_text(
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _valid_sha256(value: object) -> bool:
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
def _safe_identifier(value: str, label: str) -> str:
if _IDENTIFIER.fullmatch(value) is None:
raise L3PointPillarsAdmissionError(f"{label} is invalid")
return value
def _utc(value: str, label: str) -> None:
if not value.endswith("Z") or "T" not in value:
raise L3PointPillarsAdmissionError(f"{label} is invalid")
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise L3PointPillarsAdmissionError(f"{label} must be an object")
return value
def _array(value: object, label: str) -> list[Any]:
if not isinstance(value, list):
raise L3PointPillarsAdmissionError(f"{label} must be an array")
return value
def _string_array(value: object, label: str) -> list[str]:
values = _array(value, label)
if any(not isinstance(item, str) or not item for item in values):
raise L3PointPillarsAdmissionError(f"{label} must contain strings")
return values
def _required_string(value: dict[str, Any], key: str) -> str:
item = value.get(key)
if not isinstance(item, str) or not item.strip():
raise L3PointPillarsAdmissionError(f"L3 {key} is invalid")
return item
src/k1link/compute/pointpillars_postprocess.py
+321
View File
@@ -0,0 +1,321 @@
"""Validated NVIDIA PointPillars candidate decoding and sample-compatible NMS."""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Final
import numpy as np
POINTPILLARS_MODEL_CLASSES: Final = ("Vehicle", "Pedestrian", "Cyclist")
POINTPILLARS_OUTPUT_FIELDS: Final = (
"x",
"y",
"z",
"length",
"width",
"height",
"yaw",
"class_id",
"score",
)
POINTPILLARS_NMS_IOU_THRESHOLD: Final = 0.01
POINTPILLARS_PRE_NMS_TOP_N: Final = 4_096
POINTPILLARS_EMBEDDED_SCORE_THRESHOLD: Final = 0.1
POINTPILLARS_MODEL_POINT_CLOUD_RANGE: Final = (
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432,
)
NVIDIA_REFERENCE_COMMIT: Final = "a540badc47812a17a94e924b537d49ad3969b5a8"
_EPSILON: Final = 1e-8
class PointPillarsPostprocessError(RuntimeError):
"""PointPillars output does not satisfy the frozen NVIDIA contract."""
@dataclass(frozen=True, slots=True)
class PointPillarsBox:
x_m: float
y_m: float
z_m: float
length_m: float
width_m: float
height_m: float
yaw_rad: float
class_id: int
model_class: str
score: float
@dataclass(frozen=True, slots=True)
class _Geometry:
corners: tuple[tuple[float, float], ...]
area: float
minimum_x: float
maximum_x: float
minimum_y: float
maximum_y: float
def decode_pointpillars_output(
output_boxes: np.ndarray,
num_boxes: np.ndarray,
*,
pre_nms_top_n: int = POINTPILLARS_PRE_NMS_TOP_N,
nms_iou_threshold: float = POINTPILLARS_NMS_IOU_THRESHOLD,
) -> tuple[PointPillarsBox, ...]:
"""Decode validated rows and reproduce the NVIDIA sample's class-agnostic NMS."""
boxes = np.asarray(output_boxes)
counts = np.asarray(num_boxes)
if boxes.shape != (1, 393_216, 9) or boxes.dtype != np.float32:
raise PointPillarsPostprocessError("PointPillars output_boxes contract changed")
if counts.shape != (1,) or counts.dtype != np.int32:
raise PointPillarsPostprocessError("PointPillars num_boxes contract changed")
count = int(counts[0])
if count < 0 or count > boxes.shape[1]:
raise PointPillarsPostprocessError("PointPillars candidate count is invalid")
if (
isinstance(pre_nms_top_n, bool)
or not isinstance(pre_nms_top_n, int)
or pre_nms_top_n < 1
or not math.isfinite(nms_iou_threshold)
or not 0.0 <= nms_iou_threshold <= 1.0
):
raise PointPillarsPostprocessError("PointPillars NMS parameters are invalid")
if count == 0:
return ()
candidates = boxes[0, :count]
if not np.isfinite(candidates).all():
raise PointPillarsPostprocessError("PointPillars candidate is non-finite")
decoded = tuple(_decode_row(row) for row in candidates)
ordered = tuple(
sorted(
decoded,
key=lambda box: box.score,
reverse=True,
)[:pre_nms_top_n]
)
return _class_agnostic_nms(ordered, nms_iou_threshold)
def oriented_bev_iou(one: PointPillarsBox, another: PointPillarsBox) -> float:
"""Return BEV IoU for two validated oriented boxes."""
one_geometry = _geometry(one)
another_geometry = _geometry(another)
overlap = _intersection_area(one_geometry, another_geometry)
denominator = one_geometry.area + another_geometry.area - overlap
return overlap / max(denominator, _EPSILON)
def oriented_3d_iou(one: PointPillarsBox, another: PointPillarsBox) -> float:
"""Return oriented 3D IoU for two center-based LiDAR-frame boxes."""
one_geometry = _geometry(one)
another_geometry = _geometry(another)
bev_overlap = _intersection_area(one_geometry, another_geometry)
one_minimum_z = one.z_m - one.height_m / 2.0
one_maximum_z = one.z_m + one.height_m / 2.0
another_minimum_z = another.z_m - another.height_m / 2.0
another_maximum_z = another.z_m + another.height_m / 2.0
height_overlap = max(
0.0,
min(one_maximum_z, another_maximum_z)
- max(one_minimum_z, another_minimum_z),
)
intersection = bev_overlap * height_overlap
one_volume = one_geometry.area * one.height_m
another_volume = another_geometry.area * another.height_m
return intersection / max(one_volume + another_volume - intersection, _EPSILON)
def _decode_row(row: np.ndarray) -> PointPillarsBox:
class_value = float(row[7])
class_id = int(round(class_value))
if (
abs(class_value - class_id) > 1e-5
or class_id < 0
or class_id >= len(POINTPILLARS_MODEL_CLASSES)
):
raise PointPillarsPostprocessError("PointPillars class id is invalid")
length_m, width_m, height_m = (float(row[index]) for index in (3, 4, 5))
score = float(row[8])
if (
length_m <= 0.0
or width_m <= 0.0
or height_m <= 0.0
or score < POINTPILLARS_EMBEDDED_SCORE_THRESHOLD
or score > 1.0
):
raise PointPillarsPostprocessError(
"PointPillars dimensions or score are invalid"
)
x_m, y_m, z_m = (float(row[index]) for index in (0, 1, 2))
return PointPillarsBox(
x_m=x_m,
y_m=y_m,
z_m=z_m,
length_m=length_m,
width_m=width_m,
height_m=height_m,
yaw_rad=float(row[6]),
class_id=class_id,
model_class=POINTPILLARS_MODEL_CLASSES[class_id],
score=score,
)
def _class_agnostic_nms(
boxes: tuple[PointPillarsBox, ...],
threshold: float,
) -> tuple[PointPillarsBox, ...]:
geometries = tuple(_geometry(box) for box in boxes)
suppressed = [False] * len(boxes)
accepted: list[PointPillarsBox] = []
for index, box in enumerate(boxes):
if suppressed[index]:
continue
accepted.append(box)
geometry = geometries[index]
for candidate_index in range(index + 1, len(boxes)):
if suppressed[candidate_index]:
continue
another = geometries[candidate_index]
if not _aabbs_overlap(geometry, another):
continue
overlap = _intersection_area(geometry, another)
iou = overlap / max(geometry.area + another.area - overlap, _EPSILON)
if iou >= threshold:
suppressed[candidate_index] = True
return tuple(accepted)
def _geometry(box: PointPillarsBox) -> _Geometry:
half_length = box.length_m / 2.0
half_width = box.width_m / 2.0
cosine = math.cos(box.yaw_rad)
sine = math.sin(box.yaw_rad)
corners = tuple(
(
box.x_m + local_x * cosine - local_y * sine,
box.y_m + local_x * sine + local_y * cosine,
)
for local_x, local_y in (
(-half_length, -half_width),
(half_length, -half_width),
(half_length, half_width),
(-half_length, half_width),
)
)
x_values = [point[0] for point in corners]
y_values = [point[1] for point in corners]
return _Geometry(
corners=corners,
area=box.length_m * box.width_m,
minimum_x=min(x_values),
maximum_x=max(x_values),
minimum_y=min(y_values),
maximum_y=max(y_values),
)
def _aabbs_overlap(one: _Geometry, another: _Geometry) -> bool:
return not (
one.maximum_x < another.minimum_x
or another.maximum_x < one.minimum_x
or one.maximum_y < another.minimum_y
or another.maximum_y < one.minimum_y
)
def _intersection_area(one: _Geometry, another: _Geometry) -> float:
if not _aabbs_overlap(one, another):
return 0.0
polygon = list(one.corners)
clip = another.corners
for index, edge_start in enumerate(clip):
edge_end = clip[(index + 1) % len(clip)]
polygon = _clip_polygon(polygon, edge_start, edge_end)
if not polygon:
return 0.0
return abs(
sum(
one_point[0] * another_point[1]
- another_point[0] * one_point[1]
for one_point, another_point in zip(
polygon,
(*polygon[1:], polygon[0]),
strict=True,
)
)
) / 2.0
def _clip_polygon(
polygon: list[tuple[float, float]],
edge_start: tuple[float, float],
edge_end: tuple[float, float],
) -> list[tuple[float, float]]:
if not polygon:
return []
result: list[tuple[float, float]] = []
previous = polygon[-1]
previous_inside = _inside(previous, edge_start, edge_end)
for current in polygon:
current_inside = _inside(current, edge_start, edge_end)
if current_inside:
if not previous_inside:
result.append(_line_intersection(previous, current, edge_start, edge_end))
result.append(current)
elif previous_inside:
result.append(_line_intersection(previous, current, edge_start, edge_end))
previous = current
previous_inside = current_inside
return result
def _inside(
point: tuple[float, float],
edge_start: tuple[float, float],
edge_end: tuple[float, float],
) -> bool:
return _cross(
edge_end[0] - edge_start[0],
edge_end[1] - edge_start[1],
point[0] - edge_start[0],
point[1] - edge_start[1],
) >= -_EPSILON
def _line_intersection(
segment_start: tuple[float, float],
segment_end: tuple[float, float],
edge_start: tuple[float, float],
edge_end: tuple[float, float],
) -> tuple[float, float]:
segment_x = segment_end[0] - segment_start[0]
segment_y = segment_end[1] - segment_start[1]
edge_x = edge_end[0] - edge_start[0]
edge_y = edge_end[1] - edge_start[1]
denominator = _cross(segment_x, segment_y, edge_x, edge_y)
if abs(denominator) <= _EPSILON:
return segment_end
offset_x = edge_start[0] - segment_start[0]
offset_y = edge_start[1] - segment_start[1]
ratio = _cross(offset_x, offset_y, edge_x, edge_y) / denominator
return (
segment_start[0] + ratio * segment_x,
segment_start[1] + ratio * segment_y,
)
def _cross(one_x: float, one_y: float, another_x: float, another_y: float) -> float:
return one_x * another_y - one_y * another_x
src/k1link/datasets/__init__.py
+14
View File
@@ -38,6 +38,14 @@ from k1link.datasets.goose_qualification import (
GroundAcceptancePolicy,
qualify_goose_ground,
)
from k1link.datasets.kitti_3d_admission import (
KITTI_3D_ADMISSION_SCHEMA,
KITTI_3D_SOURCE_ID,
Kitti3DAdmissionError,
admit_kitti_3d_object_release,
read_kitti_3d_admission,
read_kitti_standard_splits,
)
from k1link.datasets.rellis_admission import (
RELLIS_ADMISSION_SCHEMA,
RellisAdmissionError,
@@ -79,6 +87,8 @@ __all__ = [
"GOOSE_QUALIFICATION_PREVIEW_SCHEMA",
"GOOSE_QUALIFICATION_PROFILE_SCHEMA",
"GOOSE_QUALIFICATION_REPORT_SCHEMA",
"KITTI_3D_ADMISSION_SCHEMA",
"KITTI_3D_SOURCE_ID",
"RELLIS_CLASSES",
"RELLIS_ADMISSION_SCHEMA",
"RELLIS_GROUND_POLICY_SCHEMA",
@@ -91,6 +101,7 @@ __all__ = [
"RellisAdmissionError",
"RellisPatchworkProfile",
"RellisSmokeError",
"Kitti3DAdmissionError",
"GoosePatchworkProfile",
"GroundAcceptancePolicy",
"DegradationProfile",
@@ -98,6 +109,7 @@ __all__ = [
"benchmark_goose_current_ground",
"benchmark_goose_patchwork_ground",
"admit_rellis_release",
"admit_kitti_3d_object_release",
"build_rellis_official_smoke_preview",
"calibrate_rellis_sensor_height",
"configured_dataset_admission_manifest",
@@ -108,6 +120,8 @@ __all__ = [
"read_dataset_admission_manifest",
"read_dataset_ground_preview",
"read_dataset_native_scan_preview",
"read_kitti_3d_admission",
"read_kitti_standard_splits",
"read_semantic_kitti_frame",
"rellis_native_scan_preview",
"qualify_rellis_ground",
src/k1link/datasets/cli.py
+21
View File
@@ -15,6 +15,10 @@ from k1link.datasets.goose_benchmark import (
)
from k1link.datasets.goose_qualification import qualify_goose_ground
from k1link.datasets.goose_review import build_goose_ground_review_pack
from k1link.datasets.kitti_3d_admission import (
Kitti3DAdmissionError,
admit_kitti_3d_object_release,
)
from k1link.datasets.rellis_admission import RellisAdmissionError, admit_rellis_release
from k1link.datasets.rellis_qualification import qualify_rellis_ground
from k1link.datasets.rellis_smoke import (
@@ -28,6 +32,23 @@ app = typer.Typer(
)
@app.command("admit-kitti-3d-object")
def admit_kitti_3d_object_command(
dataset_root: Annotated[
Path,
typer.Option("--dataset-root", exists=True, file_okay=False, resolve_path=True),
],
) -> None:
"""Verify the archive-only KITTI 3D release and standard validation split."""
try:
manifest = admit_kitti_3d_object_release(dataset_root)
except Kitti3DAdmissionError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=2) from exc
typer.echo(json.dumps(manifest, ensure_ascii=False, sort_keys=True))
@app.command("admit-goose-validation")
def admit_goose_validation_command(
dataset_root: Annotated[
src/k1link/datasets/kitti_3d_admission.py
+552
View File
@@ -0,0 +1,552 @@
"""Fail-closed admission of the KITTI 3D object development release.
The release is admitted as independent oriented-3D-box truth for the L3
PointPillars baseline. Source archives stay on Worker 006. This module does
not extract data, install a model, run inference, or authorize a K1 quality
claim.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
import re
import tempfile
import zipfile
from collections import Counter
from contextlib import suppress
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Final
KITTI_3D_ADMISSION_SCHEMA: Final = "missioncore.kitti-3d-object-admission/v1"
KITTI_3D_SOURCE_ID: Final = "kitti-3d-object/v2017"
KITTI_3D_RELEASE_ROOT: Final = "kitti-3d-object/v2017"
KITTI_3D_LICENSE: Final = "CC-BY-NC-SA-3.0"
KITTI_3D_LICENSE_URL: Final = "https://www.cvlibs.net/datasets/kitti/index.php"
KITTI_3D_BENCHMARK_URL: Final = (
"https://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d"
)
KITTI_VELODYNE_ARCHIVE: Final = "data_object_velodyne.zip"
KITTI_LABEL_ARCHIVE: Final = "data_object_label_2.zip"
KITTI_CALIB_ARCHIVE: Final = "data_object_calib.zip"
KITTI_ARCHIVE_URLS: Final[dict[str, str]] = {
KITTI_VELODYNE_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_velodyne.zip"
),
KITTI_LABEL_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_label_2.zip"
),
KITTI_CALIB_ARCHIVE: (
"https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_calib.zip"
),
}
KITTI_ARCHIVE_BYTES: Final[dict[str, int]] = {
KITTI_VELODYNE_ARCHIVE: 28_750_710_812,
KITTI_LABEL_ARCHIVE: 5_601_213,
KITTI_CALIB_ARCHIVE: 26_854_811,
}
KITTI_TRAINING_FRAMES: Final = 7_481
KITTI_TEST_FRAMES: Final = 7_518
KITTI_TARGET_CLASSES: Final = ("Car", "Pedestrian", "Cyclist")
KITTI_STANDARD_SPLIT_COMMIT: Final = (
"233f849829b6ac19afb8af8837a0246890908755"
)
KITTI_STANDARD_SPLIT_URL: Final = (
"https://github.com/open-mmlab/OpenPCDet/tree/"
f"{KITTI_STANDARD_SPLIT_COMMIT}/data/kitti/ImageSets"
)
KITTI_SPLIT_FILES: Final[dict[str, str]] = {
"train": "train.txt",
"validation": "val.txt",
}
KITTI_SPLIT_COUNTS: Final[dict[str, int]] = {
"train": 3_712,
"validation": 3_769,
}
KITTI_SPLIT_SHA256: Final[dict[str, str]] = {
"train": "b6417a1d9b18c8fdb085128e633d28ff321b7674a6d1b3841b8f43d865b281cb",
"validation": (
"657ac4bcc1e156e5b106a4ca18e1f88e012787ea1d2b5d0adeea97fee903fa86"
),
}
MAX_ARCHIVE_ENTRIES: Final = 40_000
MAX_UNCOMPRESSED_BYTES: Final = 256 * 1024**3
MAX_LABEL_MEMBER_BYTES: Final = 8 * 1024**2
_FRAME_ID = re.compile(r"^[0-9]{6}$")
_VELODYNE_MEMBER = re.compile(
r"^(?P<split>training|testing)/velodyne/(?P<frame>[0-9]{6})\.bin$"
)
_LABEL_MEMBER = re.compile(r"^training/label_2/(?P<frame>[0-9]{6})\.txt$")
_CALIB_MEMBER = re.compile(
r"^(?P<split>training|testing)/calib/(?P<frame>[0-9]{6})\.txt$"
)
_CALIB_KEYS: Final = {
"P0",
"P1",
"P2",
"P3",
"R0_rect",
"Tr_velo_to_cam",
"Tr_imu_to_velo",
}
class Kitti3DAdmissionError(RuntimeError):
"""The KITTI 3D development release violates its pinned contract."""
def admit_kitti_3d_object_release(
dataset_root: Path,
*,
velodyne_archive: Path | None = None,
label_archive: Path | None = None,
calib_archive: Path | None = None,
train_split: Path | None = None,
validation_split: Path | None = None,
) -> dict[str, Any]:
"""Verify the archive-only KITTI release and publish a path-free state."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise Kitti3DAdmissionError(
"KITTI admission requires the canonical Worker 006 D dataset root"
)
archive_root = root / KITTI_3D_RELEASE_ROOT / "archives"
split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / (
f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}"
)
archive_paths = {
KITTI_VELODYNE_ARCHIVE: _resolved_input(
archive_root, velodyne_archive, KITTI_VELODYNE_ARCHIVE
),
KITTI_LABEL_ARCHIVE: _resolved_input(
archive_root, label_archive, KITTI_LABEL_ARCHIVE
),
KITTI_CALIB_ARCHIVE: _resolved_input(
archive_root, calib_archive, KITTI_CALIB_ARCHIVE
),
}
split_paths = {
"train": _resolved_input(
split_root, train_split, KITTI_SPLIT_FILES["train"]
),
"validation": _resolved_input(
split_root, validation_split, KITTI_SPLIT_FILES["validation"]
),
}
if any(not path.is_file() for path in (*archive_paths.values(), *split_paths.values())):
raise Kitti3DAdmissionError("one or more pinned KITTI artifacts are unavailable")
archives: dict[str, dict[str, Any]] = {}
for filename, path in archive_paths.items():
size_bytes = path.stat().st_size
if size_bytes != KITTI_ARCHIVE_BYTES[filename]:
raise Kitti3DAdmissionError(
f"{filename} size differs from the pinned KITTI release"
)
archives[filename] = {
"filename": filename,
"source_url": KITTI_ARCHIVE_URLS[filename],
"size_bytes": size_bytes,
"sha256": _sha256_file(path),
"vendor_checksum_available": False,
}
splits = _read_standard_splits(split_paths)
try:
with (
zipfile.ZipFile(archive_paths[KITTI_VELODYNE_ARCHIVE]) as points_zip,
zipfile.ZipFile(archive_paths[KITTI_LABEL_ARCHIVE]) as labels_zip,
zipfile.ZipFile(archive_paths[KITTI_CALIB_ARCHIVE]) as calib_zip,
):
point_members = _member_index(points_zip)
label_members = _member_index(labels_zip)
calib_members = _member_index(calib_zip)
training_points, testing_points = _validate_velodyne(point_members)
training_labels, target_counts = _validate_labels(
labels_zip, label_members
)
training_calib, testing_calib = _validate_calibrations(
calib_zip, calib_members
)
except (OSError, KeyError, UnicodeDecodeError, zipfile.BadZipFile) as exc:
raise Kitti3DAdmissionError("KITTI archives could not be verified") from exc
training_ids = set(training_points)
if (
set(training_labels) != training_ids
or set(training_calib) != training_ids
or set(testing_points) != set(testing_calib)
):
raise Kitti3DAdmissionError("KITTI point, label, and calibration indices diverge")
split_union = set(splits["train"]) | set(splits["validation"])
if (
set(splits["train"]).intersection(splits["validation"])
or split_union != training_ids
):
raise Kitti3DAdmissionError(
"OpenPCDet train/validation split is overlapping or incomplete"
)
validation_target_counts = _target_counts_for_frames(
archive_paths[KITTI_LABEL_ARCHIVE],
set(splits["validation"]),
)
if any(validation_target_counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES):
raise Kitti3DAdmissionError("KITTI validation split lacks a target class")
identity = {
"source_id": KITTI_3D_SOURCE_ID,
"archives": archives,
"split_source": {
"repository_commit": KITTI_STANDARD_SPLIT_COMMIT,
"source_url": KITTI_STANDARD_SPLIT_URL,
"sha256": KITTI_SPLIT_SHA256,
},
"license": {
"spdx": KITTI_3D_LICENSE,
"source_url": KITTI_3D_LICENSE_URL,
"use_scope": "academic-non-commercial",
},
"representation": {
"point_fields": ["x", "y", "z", "intensity"],
"ground_truth": "oriented-3d-boxes",
"box_coordinate_frame": "camera-rectified",
"calibration_to_sensor_frame_present": True,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
manifest = {
"schema_version": KITTI_3D_ADMISSION_SCHEMA,
"source_id": KITTI_3D_SOURCE_ID,
"observed_at_utc": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
"status": "archive-ready",
"release_identity_sha256": identity_sha256,
"storage": {
"policy": "worker-d-only",
"admitted": True,
"canonical_root": True,
"path_exposed": False,
"source_archives_extracted": False,
},
"identity": identity,
"alignment": {
"training_frame_count": len(training_points),
"test_frame_count": len(testing_points),
"training_label_count": len(training_labels),
"training_calibration_count": len(training_calib),
"test_calibration_count": len(testing_calib),
"split_counts": KITTI_SPLIT_COUNTS,
"split_union_complete": True,
"split_overlap_count": 0,
"all_target_box_counts": dict(sorted(target_counts.items())),
"validation_target_box_counts": dict(
sorted(validation_target_counts.items())
),
},
"benchmark_contract": {
"independent_ground_truth": True,
"annotations": ["oriented-3d-boxes"],
"point_fields": ["x", "y", "z", "intensity"],
"eligible_split": "validation",
"target_classes": list(KITTI_TARGET_CLASSES),
"official_test_submission_authorized": False,
"retuning_on_validation_allowed": False,
"k1_quality_claim_authorized": False,
},
"next_action": "promote-staged-engine-then-stream-standard-validation",
}
_atomic_json(root / "state/kitti-3d-object-v2017.json", manifest)
return manifest
def read_kitti_3d_admission(dataset_root: Path) -> dict[str, Any]:
"""Read the current path-free state and validate its content identity."""
root = dataset_root.expanduser().absolute()
if not _is_worker_dataset_root(root):
raise Kitti3DAdmissionError(
"KITTI admission requires the canonical Worker 006 D dataset root"
)
path = root / "state/kitti-3d-object-v2017.json"
try:
manifest = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise Kitti3DAdmissionError("KITTI admission state is unavailable") from exc
if not isinstance(manifest, dict):
raise Kitti3DAdmissionError("KITTI admission state is not an object")
identity = manifest.get("identity")
if (
manifest.get("schema_version") != KITTI_3D_ADMISSION_SCHEMA
or manifest.get("source_id") != KITTI_3D_SOURCE_ID
or manifest.get("status") != "archive-ready"
or not isinstance(identity, dict)
or manifest.get("release_identity_sha256")
!= hashlib.sha256(_canonical_json(identity)).hexdigest()
):
raise Kitti3DAdmissionError("KITTI admission state identity is invalid")
return manifest
def read_kitti_standard_splits(
dataset_root: Path,
) -> dict[str, tuple[str, ...]]:
"""Read the pinned OpenPCDet split files after validating current state."""
root = dataset_root.expanduser().absolute()
read_kitti_3d_admission(root)
split_root = root / KITTI_3D_RELEASE_ROOT / "splits" / (
f"openpcdet-{KITTI_STANDARD_SPLIT_COMMIT}"
)
return _read_standard_splits(
{
split_name: split_root / filename
for split_name, filename in KITTI_SPLIT_FILES.items()
}
)
def _resolved_input(root: Path, provided: Path | None, filename: str) -> Path:
return provided.expanduser().absolute() if provided is not None else root / filename
def _is_worker_dataset_root(root: Path) -> bool:
normalized = str(root).replace("\\", "/").rstrip("/").lower()
return normalized == "/mnt/d/ndc_missioncore/datasets"
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
try:
with path.open("rb") as source:
for chunk in iter(lambda: source.read(8 * 1024**2), b""):
digest.update(chunk)
except OSError as exc:
raise Kitti3DAdmissionError("KITTI artifact cannot be hashed") from exc
return digest.hexdigest()
def _read_standard_splits(paths: dict[str, Path]) -> dict[str, tuple[str, ...]]:
result: dict[str, tuple[str, ...]] = {}
for split_name, path in paths.items():
if _sha256_file(path) != KITTI_SPLIT_SHA256[split_name]:
raise Kitti3DAdmissionError(
f"KITTI {split_name} split differs from the pinned OpenPCDet commit"
)
try:
rows = tuple(
row.strip()
for row in path.read_text(encoding="ascii").splitlines()
if row.strip()
)
except (OSError, UnicodeDecodeError) as exc:
raise Kitti3DAdmissionError("KITTI split cannot be read") from exc
if (
len(rows) != KITTI_SPLIT_COUNTS[split_name]
or len(set(rows)) != len(rows)
or any(_FRAME_ID.fullmatch(row) is None for row in rows)
):
raise Kitti3DAdmissionError(f"KITTI {split_name} split is invalid")
result[split_name] = rows
return result
def _member_index(source: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]:
members = source.infolist()
if not members or len(members) > MAX_ARCHIVE_ENTRIES:
raise Kitti3DAdmissionError("KITTI archive entry count is invalid")
total_uncompressed = 0
indexed: dict[str, zipfile.ZipInfo] = {}
for member in members:
path = PurePosixPath(member.filename)
if (
path.is_absolute()
or ".." in path.parts
or "\\" in member.filename
or member.file_size < 0
or member.compress_size < 0
):
raise Kitti3DAdmissionError("KITTI archive contains an unsafe member")
total_uncompressed += member.file_size
if total_uncompressed > MAX_UNCOMPRESSED_BYTES:
raise Kitti3DAdmissionError("KITTI archive expands beyond the admitted limit")
if member.is_dir():
continue
normalized = path.as_posix()
if normalized in indexed:
raise Kitti3DAdmissionError("KITTI archive contains duplicate members")
indexed[normalized] = member
return indexed
def _validate_velodyne(
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]:
indexed: dict[str, dict[str, zipfile.ZipInfo]] = {
"training": {},
"testing": {},
}
for path, member in members.items():
match = _VELODYNE_MEMBER.fullmatch(path)
if match is None:
continue
if member.file_size <= 0 or member.file_size % 16:
raise Kitti3DAdmissionError("KITTI Velodyne frame is not packed XYZI")
indexed[match.group("split")][match.group("frame")] = member
if (
len(indexed["training"]) != KITTI_TRAINING_FRAMES
or len(indexed["testing"]) != KITTI_TEST_FRAMES
):
raise Kitti3DAdmissionError("KITTI Velodyne frame count is invalid")
return indexed["training"], indexed["testing"]
def _validate_labels(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], Counter[str]]:
indexed: dict[str, zipfile.ZipInfo] = {}
counts: Counter[str] = Counter()
for path, member in members.items():
match = _LABEL_MEMBER.fullmatch(path)
if match is None:
continue
if member.file_size > MAX_LABEL_MEMBER_BYTES:
raise Kitti3DAdmissionError("KITTI label member is unexpectedly large")
frame_id = match.group("frame")
indexed[frame_id] = member
counts.update(_parse_label_member(source.read(member)))
if len(indexed) != KITTI_TRAINING_FRAMES:
raise Kitti3DAdmissionError("KITTI label frame count is invalid")
if any(counts[class_name] <= 0 for class_name in KITTI_TARGET_CLASSES):
raise Kitti3DAdmissionError("KITTI release lacks a target 3D box class")
return indexed, counts
def _parse_label_member(payload: bytes) -> Counter[str]:
try:
text = payload.decode("ascii")
except UnicodeDecodeError as exc:
raise Kitti3DAdmissionError("KITTI label member is not ASCII") from exc
counts: Counter[str] = Counter()
for raw_line in text.splitlines():
fields = raw_line.split()
if not fields:
continue
if len(fields) != 15:
raise Kitti3DAdmissionError("KITTI label row does not have 15 fields")
class_name = fields[0]
try:
values = [float(value) for value in fields[1:]]
except ValueError as exc:
raise Kitti3DAdmissionError("KITTI label row contains invalid numbers") from exc
if not all(math.isfinite(value) for value in values):
raise Kitti3DAdmissionError("KITTI label row contains non-finite numbers")
if class_name in KITTI_TARGET_CLASSES:
height, width, length = values[7:10]
if height <= 0 or width <= 0 or length <= 0:
raise Kitti3DAdmissionError("KITTI target box has invalid dimensions")
counts[class_name] += 1
return counts
def _validate_calibrations(
source: zipfile.ZipFile,
members: dict[str, zipfile.ZipInfo],
) -> tuple[dict[str, zipfile.ZipInfo], dict[str, zipfile.ZipInfo]]:
indexed: dict[str, dict[str, zipfile.ZipInfo]] = {
"training": {},
"testing": {},
}
for path, member in members.items():
match = _CALIB_MEMBER.fullmatch(path)
if match is None:
continue
payload = source.read(member)
try:
lines = payload.decode("ascii").splitlines()
except UnicodeDecodeError as exc:
raise Kitti3DAdmissionError("KITTI calibration is not ASCII") from exc
keys: set[str] = set()
for line in lines:
if not line.strip():
continue
key, separator, raw_values = line.partition(":")
if not separator:
raise Kitti3DAdmissionError("KITTI calibration row is invalid")
try:
values = [float(value) for value in raw_values.split()]
except ValueError as exc:
raise Kitti3DAdmissionError(
"KITTI calibration contains invalid numbers"
) from exc
if not values or not all(math.isfinite(value) for value in values):
raise Kitti3DAdmissionError(
"KITTI calibration contains non-finite numbers"
)
keys.add(key)
if not _CALIB_KEYS.issubset(keys):
raise Kitti3DAdmissionError("KITTI calibration lacks required transforms")
indexed[match.group("split")][match.group("frame")] = member
if (
len(indexed["training"]) != KITTI_TRAINING_FRAMES
or len(indexed["testing"]) != KITTI_TEST_FRAMES
):
raise Kitti3DAdmissionError("KITTI calibration frame count is invalid")
return indexed["training"], indexed["testing"]
def _target_counts_for_frames(
labels_archive: Path,
frame_ids: set[str],
) -> Counter[str]:
counts: Counter[str] = Counter()
try:
with zipfile.ZipFile(labels_archive) as source:
members = _member_index(source)
for frame_id in sorted(frame_ids):
path = f"training/label_2/{frame_id}.txt"
member = members.get(path)
if member is None:
raise Kitti3DAdmissionError(
"KITTI validation split references a missing label"
)
counts.update(_parse_label_member(source.read(member)))
except (OSError, zipfile.BadZipFile) as exc:
raise Kitti3DAdmissionError(
"KITTI validation labels could not be verified"
) from exc
return counts
def _canonical_json(payload: Any) -> bytes:
return json.dumps(
payload,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _atomic_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(
dir=path.parent,
prefix=f".{path.name}.",
suffix=".tmp",
)
try:
with os.fdopen(descriptor, "wb") as target:
target.write(_canonical_json(payload) + b"\n")
target.flush()
os.fsync(target.fileno())
os.replace(temporary, path)
except BaseException:
with suppress(OSError):
os.unlink(temporary)
raise
tests/test_kitti_3d_admission.py
+230
View File
@@ -0,0 +1,230 @@
from __future__ import annotations
import hashlib
import json
import zipfile
from pathlib import Path
from typing import Any
import pytest
from k1link.datasets import kitti_3d_admission as module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _calibration(*, complete: bool = True) -> str:
rows = [
"P0: " + " ".join(["1"] * 12),
"P1: " + " ".join(["1"] * 12),
"P2: " + " ".join(["1"] * 12),
"P3: " + " ".join(["1"] * 12),
"R0_rect: " + " ".join(["1"] * 9),
"Tr_velo_to_cam: " + " ".join(["1"] * 12),
]
if complete:
rows.append("Tr_imu_to_velo: " + " ".join(["1"] * 12))
return "\n".join(rows) + "\n"
def _label(class_name: str, *, valid: bool = True) -> str:
dimensions = "1.5 1.6 3.8" if valid else "0 1.6 3.8"
return (
f"{class_name} 0 0 0 0 0 10 10 {dimensions} 1 1 10 0\n"
)
def _release(
root: Path,
monkeypatch: pytest.MonkeyPatch,
*,
labels: dict[str, str] | None = None,
complete_calibration: bool = True,
invalid_velodyne_frame: bool = False,
train_rows: tuple[str, ...] = ("000000",),
validation_rows: tuple[str, ...] = ("000001", "000002"),
) -> dict[str, Path]:
archive_root = root / module.KITTI_3D_RELEASE_ROOT / "archives"
split_root = root / module.KITTI_3D_RELEASE_ROOT / "splits" / (
f"openpcdet-{module.KITTI_STANDARD_SPLIT_COMMIT}"
)
archive_root.mkdir(parents=True)
split_root.mkdir(parents=True)
paths = {
module.KITTI_VELODYNE_ARCHIVE: archive_root
/ module.KITTI_VELODYNE_ARCHIVE,
module.KITTI_LABEL_ARCHIVE: archive_root / module.KITTI_LABEL_ARCHIVE,
module.KITTI_CALIB_ARCHIVE: archive_root / module.KITTI_CALIB_ARCHIVE,
"train": split_root / "train.txt",
"validation": split_root / "val.txt",
}
with zipfile.ZipFile(paths[module.KITTI_VELODYNE_ARCHIVE], "w") as archive:
for frame_id in ("000000", "000001", "000002"):
size = 15 if invalid_velodyne_frame and frame_id == "000001" else 16
archive.writestr(f"training/velodyne/{frame_id}.bin", b"\x00" * size)
for frame_id in ("000000", "000001"):
archive.writestr(f"testing/velodyne/{frame_id}.bin", b"\x00" * 16)
label_payloads = labels or {
"000000": _label("Car"),
"000001": _label("Pedestrian") + _label("Car"),
"000002": _label("Cyclist"),
}
with zipfile.ZipFile(paths[module.KITTI_LABEL_ARCHIVE], "w") as archive:
for frame_id, payload in label_payloads.items():
archive.writestr(f"training/label_2/{frame_id}.txt", payload)
with zipfile.ZipFile(paths[module.KITTI_CALIB_ARCHIVE], "w") as archive:
for split, frames in {
"training": ("000000", "000001", "000002"),
"testing": ("000000", "000001"),
}.items():
for frame_id in frames:
archive.writestr(
f"{split}/calib/{frame_id}.txt",
_calibration(complete=complete_calibration),
)
paths["train"].write_text("\n".join(train_rows) + "\n", encoding="ascii")
paths["validation"].write_text(
"\n".join(validation_rows) + "\n",
encoding="ascii",
)
monkeypatch.setattr(module, "KITTI_TRAINING_FRAMES", 3)
monkeypatch.setattr(module, "KITTI_TEST_FRAMES", 2)
monkeypatch.setattr(
module,
"KITTI_ARCHIVE_BYTES",
{
name: paths[name].stat().st_size
for name in (
module.KITTI_VELODYNE_ARCHIVE,
module.KITTI_LABEL_ARCHIVE,
module.KITTI_CALIB_ARCHIVE,
)
},
)
monkeypatch.setattr(
module,
"KITTI_SPLIT_COUNTS",
{"train": len(train_rows), "validation": len(validation_rows)},
)
monkeypatch.setattr(
module,
"KITTI_SPLIT_SHA256",
{"train": _sha256(paths["train"]), "validation": _sha256(paths["validation"])},
)
monkeypatch.setattr(module, "_is_worker_dataset_root", lambda _root: True)
return paths
def test_admits_archive_only_box_truth_with_path_free_state(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(tmp_path, monkeypatch)
manifest = module.admit_kitti_3d_object_release(tmp_path)
assert manifest["status"] == "archive-ready"
assert manifest["storage"]["source_archives_extracted"] is False
assert manifest["benchmark_contract"] == {
"independent_ground_truth": True,
"annotations": ["oriented-3d-boxes"],
"point_fields": ["x", "y", "z", "intensity"],
"eligible_split": "validation",
"target_classes": ["Car", "Pedestrian", "Cyclist"],
"official_test_submission_authorized": False,
"retuning_on_validation_allowed": False,
"k1_quality_claim_authorized": False,
}
assert manifest["alignment"]["validation_target_box_counts"] == {
"Car": 1,
"Cyclist": 1,
"Pedestrian": 1,
}
serialized = json.dumps(manifest)
assert str(tmp_path) not in serialized
assert module.read_kitti_3d_admission(tmp_path) == manifest
assert module.read_kitti_standard_splits(tmp_path) == {
"train": ("000000",),
"validation": ("000001", "000002"),
}
def test_rejects_tampered_standard_split(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
paths = _release(tmp_path, monkeypatch)
paths["validation"].write_text("000002\n000001\n", encoding="ascii")
with pytest.raises(module.Kitti3DAdmissionError, match="pinned OpenPCDet"):
module.admit_kitti_3d_object_release(tmp_path)
def test_rejects_overlapping_split(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(
tmp_path,
monkeypatch,
train_rows=("000000", "000001"),
validation_rows=("000001", "000002"),
)
with pytest.raises(module.Kitti3DAdmissionError, match="overlapping or incomplete"):
module.admit_kitti_3d_object_release(tmp_path)
def test_rejects_non_xyzi_velodyne_frame(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(tmp_path, monkeypatch, invalid_velodyne_frame=True)
with pytest.raises(module.Kitti3DAdmissionError, match="not packed XYZI"):
module.admit_kitti_3d_object_release(tmp_path)
def test_rejects_invalid_target_box(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(
tmp_path,
monkeypatch,
labels={
"000000": _label("Car"),
"000001": _label("Pedestrian", valid=False),
"000002": _label("Cyclist"),
},
)
with pytest.raises(module.Kitti3DAdmissionError, match="invalid dimensions"):
module.admit_kitti_3d_object_release(tmp_path)
def test_rejects_missing_calibration_transform(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(tmp_path, monkeypatch, complete_calibration=False)
with pytest.raises(module.Kitti3DAdmissionError, match="required transforms"):
module.admit_kitti_3d_object_release(tmp_path)
def test_read_rejects_tampered_content_identity(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
_release(tmp_path, monkeypatch)
module.admit_kitti_3d_object_release(tmp_path)
state = tmp_path / "state/kitti-3d-object-v2017.json"
payload: dict[str, Any] = json.loads(state.read_text(encoding="utf-8"))
payload["identity"]["license"]["spdx"] = "unknown"
state.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(module.Kitti3DAdmissionError, match="identity is invalid"):
module.read_kitti_3d_admission(tmp_path)
tests/test_kitti_pointpillars_benchmark.py
+258
View File
@@ -0,0 +1,258 @@
from __future__ import annotations
import math
import zipfile
from pathlib import Path
import pytest
from k1link.compute.kitti_pointpillars_benchmark import (
KittiLidarTruth,
KittiPointPillarsBenchmarkError,
PointPillarsFramePrediction,
evaluate_pointpillars_predictions,
read_kitti_validation_truth,
)
from k1link.compute.pointpillars_postprocess import PointPillarsBox
def _calibration() -> str:
return "\n".join(
[
"R0_rect: 1 0 0 0 1 0 0 0 1",
"Tr_velo_to_cam: 1 0 0 0 0 1 0 0 0 0 1 0",
"",
"",
]
)
def _label(class_name: str, x_m: float) -> str:
return f"{class_name} 0 0 0 0 0 10 10 1.5 2 4 {x_m} 0 0 0\n"
def _truth(
frame_id: str,
class_name: str,
*,
x_m: float = 10.0,
) -> KittiLidarTruth:
return KittiLidarTruth(
frame_id=frame_id,
benchmark_class=class_name,
x_m=x_m,
y_m=0.0,
z_m=0.0,
length_m=4.0,
width_m=2.0,
height_m=1.5,
yaw_rad=0.0,
)
def _prediction_box(
model_class: str,
*,
x_m: float = 10.0,
score: float = 0.9,
) -> PointPillarsBox:
return PointPillarsBox(
x_m=x_m,
y_m=0.0,
z_m=0.0,
length_m=4.0,
width_m=2.0,
height_m=1.5,
yaw_rad=0.0,
class_id={"Vehicle": 0, "Pedestrian": 1, "Cyclist": 2}[model_class],
model_class=model_class,
score=score,
)
def _perfect_fixture() -> tuple[
dict[str, tuple[KittiLidarTruth, ...]],
tuple[PointPillarsFramePrediction, ...],
]:
classes = (
("000000", "Car", "Vehicle"),
("000001", "Pedestrian", "Pedestrian"),
("000002", "Cyclist", "Cyclist"),
)
truth = {
frame_id: (_truth(frame_id, benchmark_class),)
for frame_id, benchmark_class, _ in classes
}
predictions = tuple(
PointPillarsFramePrediction(
frame_id=frame_id,
boxes=(_prediction_box(model_class),),
inference_ms=50.0 + index,
)
for index, (frame_id, _, model_class) in enumerate(classes)
)
return truth, predictions
def test_reads_and_converts_kitti_camera_bottom_centers(
tmp_path: Path,
) -> None:
labels = tmp_path / "labels.zip"
calibrations = tmp_path / "calib.zip"
with zipfile.ZipFile(labels, "w") as archive:
archive.writestr(
"training/label_2/000000.txt",
_label("Car", 10.0),
)
archive.writestr(
"training/label_2/000001.txt",
_label("Pedestrian", 11.0),
)
archive.writestr(
"training/label_2/000002.txt",
_label("Cyclist", 12.0),
)
with zipfile.ZipFile(calibrations, "w") as archive:
for frame_id in ("000000", "000001", "000002"):
archive.writestr(
f"training/calib/{frame_id}.txt",
_calibration(),
)
truth = read_kitti_validation_truth(
labels_archive=labels,
calibrations_archive=calibrations,
validation_frame_ids=("000000", "000001", "000002"),
)
car = truth["000000"][0]
assert car.x_m == 10.0
assert car.z_m == pytest.approx(0.75)
assert car.yaw_rad == pytest.approx(-math.pi / 2.0)
assert car.length_m == 4.0
assert car.width_m == 2.0
def test_perfect_predictions_produce_complete_metrics() -> None:
truth, predictions = _perfect_fixture()
report = evaluate_pointpillars_predictions(
truth_by_frame=truth,
predictions=predictions,
)
assert report["aggregates"]["bev_map40"] == pytest.approx(1.0)
assert report["aggregates"]["3d_map40"] == pytest.approx(1.0)
assert report["aggregates"]["false_occupied_rate"] == 0.0
assert report["aggregates"]["center_error_m"]["mean"] == 0.0
assert report["aggregates"]["range_error_m"]["mean"] == 0.0
assert report["aggregates"]["yaw_error_rad"]["mean"] == 0.0
assert report["aggregates"]["distance_bucket_recall"]["0-20m"]["recall"] == 1.0
assert report["metric_contract"]["evaluation_kind"] == (
"public-cross-domain-transfer-probe"
)
assert report["claim_boundary"]["native_model_accuracy_evaluated"] is False
assert report["claim_boundary"]["k1_transfer_evaluated"] is False
def test_false_prediction_reduces_precision_and_counts_false_occupied() -> None:
truth, predictions = _perfect_fixture()
first = predictions[0]
predictions = (
PointPillarsFramePrediction(
frame_id=first.frame_id,
boxes=(
_prediction_box("Vehicle", x_m=40.0, score=0.95),
*first.boxes,
),
inference_ms=first.inference_ms,
),
*predictions[1:],
)
report = evaluate_pointpillars_predictions(
truth_by_frame=truth,
predictions=predictions,
)
assert report["per_class"]["Car"]["true_positives"] == 1
assert report["per_class"]["Car"]["false_positives"] == 1
assert report["per_class"]["Car"]["precision"] == pytest.approx(0.5)
assert report["aggregates"]["false_occupied_rate"] == pytest.approx(0.25)
def test_predictions_outside_shared_cross_domain_range_are_not_false_positives() -> None:
truth, predictions = _perfect_fixture()
first = predictions[0]
predictions = (
PointPillarsFramePrediction(
frame_id=first.frame_id,
boxes=(
_prediction_box("Vehicle", x_m=-10.0, score=0.95),
*first.boxes,
),
inference_ms=first.inference_ms,
),
*predictions[1:],
)
report = evaluate_pointpillars_predictions(
truth_by_frame=truth,
predictions=predictions,
)
assert report["per_class"]["Car"]["false_positives"] == 0
assert report["aggregates"]["prediction_volume"] == {
"model_output_box_count": 4,
"evaluated_box_count": 3,
"outside_shared_range_count": 1,
}
def test_frame_set_must_equal_admitted_validation_split() -> None:
truth, predictions = _perfect_fixture()
with pytest.raises(
KittiPointPillarsBenchmarkError,
match="do not equal",
):
evaluate_pointpillars_predictions(
truth_by_frame=truth,
predictions=predictions[:-1],
)
def test_unknown_model_class_is_rejected() -> None:
truth, predictions = _perfect_fixture()
first = predictions[0]
unknown = PointPillarsBox(
**{
field: getattr(first.boxes[0], field)
for field in (
"x_m",
"y_m",
"z_m",
"length_m",
"width_m",
"height_m",
"yaw_rad",
"class_id",
"score",
)
},
model_class="Unknown",
)
predictions = (
PointPillarsFramePrediction(
frame_id=first.frame_id,
boxes=(unknown,),
inference_ms=first.inference_ms,
),
*predictions[1:],
)
with pytest.raises(KittiPointPillarsBenchmarkError, match="not admitted"):
evaluate_pointpillars_predictions(
truth_by_frame=truth,
predictions=predictions,
)
tests/test_l3_pointpillars_admission.py
+428
View File
@@ -0,0 +1,428 @@
from __future__ import annotations
import copy
import json
from pathlib import Path
from typing import Any
import pytest
from k1link.compute.l3_pointpillars_admission import (
L3PointPillarsAdmissionError,
build_l3_pointpillars_admission,
read_l3_pointpillars_admission,
)
SHA = "a" * 64
TRITON_SHA = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
def _profile() -> dict[str, Any]:
return {
"schema_version": "missioncore.l3-pointpillars-benchmark-profile/v1",
"profile_id": "l3-pointpillars-public-transfer-probe-v1",
"detector": {
"family": "nvidia-tao-pointpillars",
"upstream_model_id": "nvidia/tao/pointpillarnet",
"candidate_frozen": False,
"candidate_model_version": None,
"candidate_source_sha256": None,
"candidate_label_sha256": None,
"triton_model_name": "pointpillars",
"required_source_format": "onnx",
"input_representation": "native-sensor-scan",
"input_coordinate_frame": "sensor/lidar",
"input_fields": ["x", "y", "z", "intensity"],
"batch_size": 1,
"maximum_points": 204_800,
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432,
],
"training_domain": "proprietary-solid-state-lidar",
"training_ground_truth_publicly_reproducible": False,
"model_classes": ["Vehicle", "Pedestrian", "Cyclist"],
"onnx_contract_sha256": (
"2fd29cd054ab058c2cfec3dfba305c71e123ef3f04b457d0c64de0c8dac2e1be"
),
"postprocessing": {
"reference_repository": (
"https://github.com/NVIDIA-AI-IOT/tao_toolkit_recipes"
),
"reference_commit": "a540badc47812a17a94e924b537d49ad3969b5a8",
"output_row_fields": [
"x",
"y",
"z",
"length",
"width",
"height",
"yaw",
"class_id",
"score",
],
"class_agnostic_nms": True,
"nms_iou_threshold": 0.01,
"pre_nms_top_n": 4096,
"embedded_score_threshold": 0.1,
"embedded_contract_source": "onnx-node-attributes",
},
},
"runtime_policy": {
"existing_triton_only": True,
"second_serving_stack_allowed": False,
"engine_built_on_target_required": True,
"precision": "strongly-typed",
"triton_image": "nvcr.io/nvidia/tritonserver:26.06-py3",
"triton_image_digest": TRITON_SHA,
},
"public_cross_domain_probe": {
"required_split": "validation",
"required_ground_truth": "oriented-3d-boxes",
"independent_ground_truth_required": True,
"benchmark_classes": ["Car", "Pedestrian", "Cyclist"],
"model_to_benchmark_class_mapping": {
"Vehicle": "Car",
"Pedestrian": "Pedestrian",
"Cyclist": "Cyclist",
},
"metrics": [
"bev-map",
"3d-map",
"center-error-m",
"range-error-m",
"yaw-error-rad",
"distance-bucket-recall",
"false-occupied-rate",
"end-to-end-latency-ms",
],
"metric_contract": {
"official_kitti_server_metric": False,
"evaluation_kind": "public-cross-domain-transfer-probe",
"ap_interpolation": "40-point",
"difficulty_filtering": False,
"predictions_outside_shared_range_ignored": True,
"iou_thresholds": {
"Car": 0.7,
"Pedestrian": 0.5,
"Cyclist": 0.5,
},
"distance_buckets_m": [[0, 20], [20, 40], [40, 70]],
},
"native_accuracy_claim_allowed": False,
"retuning_allowed": False,
},
"k1_transfer_stability": {
"requires_completed_public_cross_domain_probe": True,
"metrics": [
"input-admission-rate",
"output-schema-valid-rate",
"deterministic-replay-rate",
"end-to-end-latency-ms",
"queue-wait-ms",
"drop-rate",
],
"accuracy_claim_allowed": False,
"retuning_allowed": False,
},
"authority": {
"shadow_only": True,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
def _semantic_dataset(dataset_id: str = "goose-3d/v2025-08-22") -> dict[str, Any]:
return {
"dataset_id": dataset_id,
"installed": True,
"release_identity_sha256": SHA,
"license": "CC-BY-SA-4.0",
"point_fields": ["x", "y", "z", "intensity"],
"annotations": ["point-semantic-labels", "point-instance-labels"],
"splits": ["validation"],
"independent_ground_truth": True,
}
def _box_dataset(*, installed: bool = True) -> dict[str, Any]:
return {
"dataset_id": "kitti-3d-object-detection/v1",
"installed": installed,
"release_identity_sha256": SHA if installed else None,
"license": "CC-BY-NC-SA-3.0",
"point_fields": ["x", "y", "z", "intensity"],
"annotations": ["oriented-3d-boxes"],
"splits": ["validation"],
"independent_ground_truth": True,
}
def _worker(*, pointpillars: bool = False) -> dict[str, Any]:
models: list[dict[str, Any]] = [
{
"name": "yolox_s",
"backend": "onnxruntime",
"artifact_sha256": SHA,
}
]
if pointpillars:
models.append(
{
"name": "pointpillars",
"upstream_version": "tao-6.26.03-test",
"source_model_sha256": "c" * 64,
"source_label_sha256": "e" * 64,
"source_format": "onnx",
"backend": "tensorrt",
"precision": "strongly-typed",
"artifact_sha256": "b" * 64,
"engine_built_on_target": True,
"provenance_verified": True,
"input_fields": ["x", "y", "z", "intensity"],
"maximum_points": 204_800,
"point_cloud_range": [
-51.20000076293945,
-51.20000076293945,
-1.399999976158142,
51.20000076293945,
51.20000076293945,
4.400000095367432,
],
"model_classes": ["Vehicle", "Pedestrian", "Cyclist"],
"outputs": [
{
"name": "output_boxes",
"dtype": "FP32",
"shape": [1, 393_216, 9],
},
{"name": "num_boxes", "dtype": "INT32", "shape": [1]},
],
"representation_smoke": {
"status": "engine-executed",
"input_artifact_sha256": "d" * 64,
"input_point_count": 169_883,
"single_query_gpu_compute_ms": 55.0,
"accuracy_evaluated": False,
"navigation_or_safety_accepted": False,
},
}
)
return {
"schema_version": "missioncore.l3-worker-inventory/v1",
"host_id": "worker-006",
"observed_at_utc": "2026-07-30T20:57:38Z",
"serving_stack_count": 1,
"staged_models": [],
"triton": {
"container_name": "ndc-mission-core-triton",
"image": "nvcr.io/nvidia/tritonserver:26.06-py3",
"image_digest": TRITON_SHA,
"healthy": True,
"strict_readiness": True,
"model_control_mode": "explicit",
"model_repository_read_only": True,
"models": models,
},
"gpu": {
"name": "NVIDIA GeForce RTX 4090",
"driver_version": "610.47",
"memory_total_mib": 24_564,
},
}
def _write(path: Path, value: object) -> None:
path.write_text(json.dumps(value), encoding="utf-8")
def _build(
tmp_path: Path,
*,
profile: dict[str, Any] | None = None,
datasets: list[dict[str, Any]] | None = None,
worker: dict[str, Any] | None = None,
):
profile_path = tmp_path / "profile.json"
datasets_path = tmp_path / "datasets.json"
worker_path = tmp_path / "worker.json"
_write(profile_path, profile or _profile())
_write(
datasets_path,
{
"schema_version": "missioncore.l3-lidar-dataset-inventory/v1",
"observed_at_utc": "2026-07-30T20:57:38Z",
"datasets": datasets or [_semantic_dataset()],
},
)
_write(worker_path, worker or _worker())
return build_l3_pointpillars_admission(
profile_path=profile_path,
dataset_inventory_path=datasets_path,
worker_inventory_path=worker_path,
output_root=tmp_path / "results",
)
def test_semantic_point_truth_and_missing_model_block_public_probe(
tmp_path: Path,
) -> None:
result = _build(tmp_path)
assert result.report["status"] == "blocked-foundation-assets"
assert result.report["blocker_codes"] == [
"public-oriented-3d-box-truth-not-admitted",
"pointpillars-compatible-candidate-not-frozen",
]
assert result.report["next_gate"] == (
"admit-public-3d-box-split-and-freeze-compatible-pointpillars-candidate"
)
assert result.public_transfer_probe_authorized is False
finding = result.report["dataset_findings"][0]
assert finding["semantic_or_instance_labels_are_not_boxes"] is True
assert finding["oriented_3d_box_accuracy_eligible"] is False
assert (
result.report["decision"]["semantic_point_labels_substitute_for_3d_boxes"]
is False
)
assert result.report["decision"]["k1_transfer_stability_authorized"] is False
def test_box_truth_and_target_built_model_authorize_public_probe_only(
tmp_path: Path,
) -> None:
profile = _profile()
profile["detector"]["candidate_frozen"] = True
profile["detector"]["candidate_model_version"] = "tao-6.26.03-test"
profile["detector"]["candidate_source_sha256"] = "c" * 64
profile["detector"]["candidate_label_sha256"] = "e" * 64
result = _build(
tmp_path,
profile=profile,
datasets=[_semantic_dataset(), _box_dataset()],
worker=_worker(pointpillars=True),
)
assert result.report["status"] == "ready-for-public-cross-domain-probe"
assert result.report["blocker_codes"] == []
assert result.report["eligible_public_probe_dataset_ids"] == [
"kitti-3d-object-detection/v1"
]
assert result.public_transfer_probe_authorized is True
assert result.report["decision"] == {
"public_cross_domain_probe_authorized": True,
"native_model_accuracy_claim_authorized": False,
"k1_transfer_stability_authorized": False,
"k1_transfer_quality_claim_authorized": False,
"semantic_point_labels_substitute_for_3d_boxes": False,
"fine_tuning_allowed": False,
"second_serving_stack_allowed": False,
"lab_publication_allowed": False,
"centerpoint_comparison_allowed": False,
}
assert result.report["next_gate"] == (
"run-public-cross-domain-pointpillars-probe"
)
def test_pointpillars_without_target_engine_provenance_remains_blocked(
tmp_path: Path,
) -> None:
profile = _profile()
profile["detector"]["candidate_frozen"] = True
profile["detector"]["candidate_model_version"] = "tao-6.26.03-test"
profile["detector"]["candidate_source_sha256"] = "c" * 64
profile["detector"]["candidate_label_sha256"] = "e" * 64
worker = _worker(pointpillars=True)
worker["triton"]["models"][1]["engine_built_on_target"] = False
result = _build(
tmp_path,
profile=profile,
datasets=[_box_dataset()],
worker=worker,
)
assert result.report["blocker_codes"] == [
"pointpillars-target-engine-or-provenance-not-verified"
]
assert result.report["next_gate"] == "verify-target-engine-and-model-provenance"
def test_verified_staged_engine_is_distinct_from_live_install(tmp_path: Path) -> None:
profile = _profile()
profile["detector"]["candidate_frozen"] = True
profile["detector"]["candidate_model_version"] = "tao-6.26.03-test"
profile["detector"]["candidate_source_sha256"] = "c" * 64
profile["detector"]["candidate_label_sha256"] = "e" * 64
worker = _worker(pointpillars=True)
staged = worker["triton"]["models"].pop()
worker["staged_models"] = [staged]
result = _build(
tmp_path,
profile=profile,
datasets=[_box_dataset(installed=False)],
worker=worker,
)
assert result.report["blocker_codes"] == [
"public-oriented-3d-box-truth-not-admitted",
"pointpillars-model-not-installed-live",
]
assert result.report["detector"]["staged_target_engine_ready"] is True
assert result.report["detector"]["model_ready"] is False
assert result.report["next_gate"] == (
"admit-public-3d-box-split-then-install-staged-pointpillars-model"
)
def test_second_serving_stack_is_rejected(tmp_path: Path) -> None:
profile = _profile()
profile["detector"]["candidate_frozen"] = True
profile["detector"]["candidate_model_version"] = "tao-6.26.03-test"
profile["detector"]["candidate_source_sha256"] = "c" * 64
profile["detector"]["candidate_label_sha256"] = "e" * 64
worker = _worker(pointpillars=True)
worker["serving_stack_count"] = 2
result = _build(
tmp_path,
profile=profile,
datasets=[_box_dataset()],
worker=worker,
)
assert result.report["blocker_codes"] == [
"canonical-triton-runtime-policy-not-satisfied"
]
assert result.report["runtime_checks"]["second_serving_stack_absent"] is False
assert result.report["decision"]["second_serving_stack_allowed"] is False
def test_profile_cannot_skip_the_public_transfer_probe(tmp_path: Path) -> None:
profile = copy.deepcopy(_profile())
profile["k1_transfer_stability"][
"requires_completed_public_cross_domain_probe"
] = False
with pytest.raises(L3PointPillarsAdmissionError):
_build(tmp_path, profile=profile)
def test_result_is_content_addressed_and_detects_tampering(tmp_path: Path) -> None:
first = _build(tmp_path)
second = _build(tmp_path)
assert first.result_id == second.result_id
report_path = first.result_root / "admission-report.json"
report = json.loads(report_path.read_text(encoding="utf-8"))
report["status"] = "changed"
_write(report_path, report)
with pytest.raises(L3PointPillarsAdmissionError):
read_l3_pointpillars_admission(first.result_root)
tests/test_l3_pointpillars_worker_package.py
+82
View File
@@ -0,0 +1,82 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
from types import ModuleType
import pytest
def _load_builder() -> ModuleType:
path = (
Path(__file__).resolve().parents[1]
/ "experiments/perception/prepare_l3_pointpillars_worker_package.py"
)
specification = importlib.util.spec_from_file_location(
"l3_pointpillars_worker_package_test",
path,
)
assert specification is not None and specification.loader is not None
module = importlib.util.module_from_spec(specification)
specification.loader.exec_module(module)
return module
def test_builds_and_reopens_minimal_content_addressed_package(
tmp_path: Path,
) -> None:
builder = _load_builder()
repository = Path(__file__).resolve().parents[1]
admission = tmp_path / (
"l3-pointpillars-admission-" + ("a" * 64)
)
admission.mkdir()
(admission / "manifest.json").write_text("{}\n", encoding="utf-8")
(admission / "admission-report.json").write_text("{}\n", encoding="utf-8")
package = builder.build_l3_worker_package(
repository_root=repository,
output_root=tmp_path / "packages",
admission_result=admission,
)
reopened = builder.build_l3_worker_package(
repository_root=repository,
output_root=tmp_path / "packages",
admission_result=admission,
)
manifest = builder.validate_l3_worker_package(package)
assert reopened == package
assert manifest["package_id"] == package.name
assert manifest["identity"]["execution_policy"] == {
"sequential": True,
"parallel_workers": 1,
"existing_triton_only": True,
"container_creation_allowed": False,
"container_restart_allowed": False,
"raw_tensor_export_allowed": False,
}
assert not (package / "runtime/k1link/compute/__pycache__").exists()
assert (
package / "runtime/k1link/datasets/kitti_3d_admission.py"
).is_file()
def test_validation_rejects_modified_member(tmp_path: Path) -> None:
builder = _load_builder()
repository = Path(__file__).resolve().parents[1]
package = builder.build_l3_worker_package(
repository_root=repository,
output_root=tmp_path / "packages",
)
target = package / "input/profile.json"
profile = json.loads(target.read_text(encoding="utf-8"))
profile["profile_id"] = "changed"
target.write_text(json.dumps(profile), encoding="utf-8")
with pytest.raises(
builder.L3WorkerPackageError,
match="artifact changed",
):
builder.validate_l3_worker_package(package)
tests/test_l3_pointpillars_worker_runtime.py
+90
View File
@@ -0,0 +1,90 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
from types import ModuleType
import numpy as np
import pytest
def _load_worker() -> ModuleType:
path = (
Path(__file__).resolve().parents[1]
/ "experiments/perception/worker/run_l3_pointpillars_public_baseline.py"
)
specification = importlib.util.spec_from_file_location(
"l3_pointpillars_worker_test",
path,
)
assert specification is not None and specification.loader is not None
module = importlib.util.module_from_spec(specification)
specification.loader.exec_module(module)
return module
class _Response:
def __init__(self, payload: bytes, header_length: int) -> None:
self._payload = payload
self.headers = {"Inference-Header-Content-Length": str(header_length)}
self.status = 200
def __enter__(self) -> _Response:
return self
def __exit__(self, *_args: object) -> None:
return None
def read(self) -> bytes:
return self._payload
def test_binary_triton_response_parses_both_outputs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
worker = _load_worker()
boxes = np.zeros((1, 393_216, 9), dtype=np.float32)
boxes[0, 0] = np.asarray(
[10.0, 0.0, 0.0, 4.0, 2.0, 1.5, 0.0, 0.0, 0.9],
dtype=np.float32,
)
count = np.asarray([1], dtype=np.int32)
count_bytes = count.tobytes()
boxes_bytes = boxes.tobytes()
header = json.dumps(
{
"outputs": [
{
"name": "num_boxes",
"datatype": "INT32",
"shape": [1],
"parameters": {"binary_data_size": len(count_bytes)},
},
{
"name": "output_boxes",
"datatype": "FP32",
"shape": [1, 393_216, 9],
"parameters": {"binary_data_size": len(boxes_bytes)},
},
]
},
separators=(",", ":"),
).encode()
payload = header + count_bytes + boxes_bytes
monkeypatch.setattr(
worker.urllib.request,
"urlopen",
lambda *_args, **_kwargs: _Response(payload, len(header)),
)
output_boxes, output_count, elapsed_ms = worker._infer(
"http://127.0.0.1:8000",
np.zeros((1, 204_800, 4), dtype=np.float32),
np.asarray([1], dtype=np.int32),
)
assert output_boxes.shape == (1, 393_216, 9)
assert output_count.tolist() == [1]
assert output_boxes[0, 0, 8] == pytest.approx(0.9)
assert elapsed_ms > 0.0
tests/test_pointpillars_postprocess.py
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import math
import numpy as np
import pytest
from k1link.compute.pointpillars_postprocess import (
PointPillarsBox,
PointPillarsPostprocessError,
decode_pointpillars_output,
oriented_3d_iou,
oriented_bev_iou,
)
def _outputs(rows: list[list[float]]) -> tuple[np.ndarray, np.ndarray]:
boxes = np.zeros((1, 393_216, 9), dtype=np.float32)
boxes[0, : len(rows)] = np.asarray(rows, dtype=np.float32)
return boxes, np.asarray([len(rows)], dtype=np.int32)
def _row(
*,
x: float,
y: float = 0.0,
length: float = 4.0,
width: float = 2.0,
yaw: float = 0.0,
class_id: int = 0,
score: float = 0.9,
) -> list[float]:
return [x, y, 0.0, length, width, 1.5, yaw, float(class_id), score]
def _box(*, yaw: float = 0.0, x: float = 0.0) -> PointPillarsBox:
return PointPillarsBox(
x_m=x,
y_m=0.0,
z_m=0.0,
length_m=4.0,
width_m=2.0,
height_m=1.5,
yaw_rad=yaw,
class_id=0,
model_class="Vehicle",
score=0.9,
)
def test_decodes_native_label_order_and_sorts_by_score() -> None:
output_boxes, num_boxes = _outputs(
[
_row(x=20.0, class_id=2, score=0.6),
_row(x=0.0, class_id=0, score=0.9),
_row(x=10.0, class_id=1, score=0.8),
]
)
decoded = decode_pointpillars_output(output_boxes, num_boxes)
assert [box.model_class for box in decoded] == [
"Vehicle",
"Pedestrian",
"Cyclist",
]
assert [box.score for box in decoded] == pytest.approx([0.9, 0.8, 0.6])
def test_nms_reproduces_nvidia_sample_class_agnostic_suppression() -> None:
output_boxes, num_boxes = _outputs(
[
_row(x=0.0, class_id=0, score=0.9),
_row(x=0.1, class_id=1, score=0.8),
_row(x=20.0, class_id=1, score=0.7),
]
)
decoded = decode_pointpillars_output(output_boxes, num_boxes)
assert [(box.x_m, box.model_class) for box in decoded] == [
(0.0, "Vehicle"),
(20.0, "Pedestrian"),
]
def test_pre_nms_cap_is_applied_after_stable_score_ordering() -> None:
output_boxes, num_boxes = _outputs(
[
_row(x=0.0, score=0.7),
_row(x=10.0, score=0.9),
_row(x=20.0, score=0.8),
]
)
decoded = decode_pointpillars_output(
output_boxes,
num_boxes,
pre_nms_top_n=2,
)
assert [box.x_m for box in decoded] == [10.0, 20.0]
def test_oriented_bev_iou_handles_rotation_and_separation() -> None:
assert oriented_bev_iou(_box(), _box()) == pytest.approx(1.0)
assert oriented_bev_iou(_box(), _box(yaw=math.pi / 2.0)) == pytest.approx(
1.0 / 3.0
)
assert oriented_bev_iou(_box(), _box(x=20.0)) == 0.0
assert oriented_3d_iou(_box(), _box()) == pytest.approx(1.0)
assert oriented_3d_iou(_box(), _box(x=20.0)) == 0.0
@pytest.mark.parametrize(
("row", "message"),
[
(_row(x=0.0, class_id=3), "class id"),
(_row(x=0.0, length=0.0), "dimensions or score"),
(_row(x=0.0, score=0.09), "dimensions or score"),
(_row(x=0.0, score=1.1), "dimensions or score"),
],
)
def test_invalid_candidate_fails_closed(row: list[float], message: str) -> None:
output_boxes, num_boxes = _outputs([row])
with pytest.raises(PointPillarsPostprocessError, match=message):
decode_pointpillars_output(output_boxes, num_boxes)
def test_output_tensor_contract_is_exact() -> None:
with pytest.raises(PointPillarsPostprocessError, match="output_boxes contract"):
decode_pointpillars_output(
np.zeros((1, 1, 9), dtype=np.float32),
np.asarray([0], dtype=np.int32),
)