From 28effdde23dd630de117849a93e820626f9d3090 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 26 Aug 2026 02:06:03 +0300 Subject: [PATCH] feat(perception): build native raw RF-DETR engine --- .../convert_m48n_rf_detr_native_onnx_fp16.py | 185 ++++++ .../export_m48n_rf_detr_native_worker.py | 156 +++++ .../run_m48n_native_parity_worker.py | 509 +++++++++++++++ .../run_m48n_native_vs_704_worker.py | 588 ++++++++++++++++++ .../Invoke-M48NNativeCandidateBuild.ps1 | 349 +++++++++++ .../worker/Invoke-M48NNativeParity.ps1 | 238 +++++++ .../worker/Invoke-M48NNativeVs704.ps1 | 275 ++++++++ .../rf_detr_large_native_kb4_config.pbtxt | 47 ++ .../wrap_m48n_rf_detr_native_uint8_onnx.py | 298 +++++++++ 9 files changed, 2645 insertions(+) create mode 100644 experiments/perception/convert_m48n_rf_detr_native_onnx_fp16.py create mode 100644 experiments/perception/export_m48n_rf_detr_native_worker.py create mode 100644 experiments/perception/run_m48n_native_parity_worker.py create mode 100644 experiments/perception/run_m48n_native_vs_704_worker.py create mode 100644 experiments/perception/worker/Invoke-M48NNativeCandidateBuild.ps1 create mode 100644 experiments/perception/worker/Invoke-M48NNativeParity.ps1 create mode 100644 experiments/perception/worker/Invoke-M48NNativeVs704.ps1 create mode 100644 experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt create mode 100644 experiments/perception/wrap_m48n_rf_detr_native_uint8_onnx.py diff --git a/experiments/perception/convert_m48n_rf_detr_native_onnx_fp16.py b/experiments/perception/convert_m48n_rf_detr_native_onnx_fp16.py new file mode 100644 index 0000000..daf8bae --- /dev/null +++ b/experiments/perception/convert_m48n_rf_detr_native_onnx_fp16.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Convert the native 608x800 RF-DETR core to a strongly typed FP16 graph.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import warnings +from pathlib import Path +from typing import Any, Final + +import onnx # type: ignore[import-not-found] +from onnx import TensorProto +from onnxconverter_common import float16 # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48n-rf-detr-native-onnx-fp16-conversion/v0" +PROFILE_ID: Final = "rf-detr-large-coco-native-kb4-608x800-trt11-fp16/v0" +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-input-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + arguments = parser.parse_args() + + source = arguments.input.resolve(strict=True) + source_sha256 = sha256_path(source) + if source_sha256 != arguments.expected_input_sha256: + raise RuntimeError("source ONNX SHA-256 does not match the export manifest") + output = arguments.output.absolute() + manifest = arguments.manifest.absolute() + if output.exists() or manifest.exists(): + raise RuntimeError("native FP16 ONNX output or manifest already exists") + + started_utc_ns = time.time_ns() + graph = onnx.load(str(source)) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"the float32 number .* will be truncated to .*", + category=UserWarning, + module=r"onnxconverter_common\.float16", + ) + converted = float16.convert_float_to_float16( + graph, + keep_io_types=False, + disable_shape_infer=False, + ) + retargeted_casts = _retarget_float_casts_to_fp16(converted) + _insert_fp32_input_cast(converted) + onnx.checker.check_model(converted) + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + onnx.save(converted, str(output)) + verified = onnx.load(str(output), load_external_data=False) + onnx.checker.check_model(verified) + inputs = [_tensor_description(value) for value in verified.graph.input] + outputs = [_tensor_description(value) for value in verified.graph.output] + expected_input = [ + {"name": "input", "element_type": TensorProto.FLOAT, "shape": [1, 3, 608, 800]} + ] + if inputs != expected_input: + raise RuntimeError(f"native FP16 ONNX input boundary changed: {inputs}") + expected_outputs = { + "dets": TensorProto.FLOAT16, + "labels": TensorProto.FLOAT16, + } + output_types = {item["name"]: item["element_type"] for item in outputs} + if output_types != expected_outputs: + raise RuntimeError(f"native FP16 ONNX outputs are not FLOAT16: {outputs}") + initializer_counts = _initializer_type_counts(verified) + if initializer_counts.get("FLOAT16", 0) == 0: + raise RuntimeError("native FP16 ONNX has no FLOAT16 initializers") + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": PROFILE_ID, + "source_onnx_sha256": source_sha256, + "output_onnx_sha256": sha256_path(output), + "output_size_bytes": output.stat().st_size, + "inputs": inputs, + "outputs": outputs, + "initializer_type_counts": initializer_counts, + "float_casts_retargeted_to_fp16": retargeted_casts, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": FALSE_AUTHORITY, + } + manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest.write_bytes(canonical_json(document) + b"\n") + print(output) + print(json.dumps(document, indent=2, sort_keys=True)) + return 0 + + +def _tensor_description(value: Any) -> dict[str, object]: + tensor = value.type.tensor_type + return { + "name": value.name, + "element_type": tensor.elem_type, + "shape": [_dimension_value(item) for item in tensor.shape.dim], + } + + +def _dimension_value(value: Any) -> int | str | None: + if value.HasField("dim_value"): + return int(value.dim_value) + if value.HasField("dim_param"): + return str(value.dim_param) + return None + + +def _initializer_type_counts(graph: Any) -> dict[str, int]: + counts: dict[str, int] = {} + for initializer in graph.graph.initializer: + name = TensorProto.DataType.Name(initializer.data_type) + counts[name] = counts.get(name, 0) + 1 + return dict(sorted(counts.items())) + + +def _insert_fp32_input_cast(graph: Any) -> None: + input_value = next((item for item in graph.graph.input if item.name == "input"), None) + if input_value is None: + raise RuntimeError("native RF-DETR graph has no input tensor named 'input'") + if input_value.type.tensor_type.elem_type != TensorProto.FLOAT16: + raise RuntimeError("native RF-DETR input is not FLOAT16 before boundary adaptation") + cast_output = "missioncore_input_fp16" + for node in graph.graph.node: + for index, name in enumerate(node.input): + if name == "input": + node.input[index] = cast_output + cast = onnx.helper.make_node( + "Cast", + inputs=["input"], + outputs=[cast_output], + name="missioncore_input_fp32_to_fp16", + to=TensorProto.FLOAT16, + ) + graph.graph.node.insert(0, cast) + input_value.type.tensor_type.elem_type = TensorProto.FLOAT + + +def _retarget_float_casts_to_fp16(graph: Any) -> int: + count = 0 + for node in graph.graph.node: + if node.op_type != "Cast": + continue + for attribute in node.attribute: + if attribute.name == "to" and attribute.i == TensorProto.FLOAT: + attribute.i = TensorProto.FLOAT16 + count += 1 + if count == 0: + raise RuntimeError("native RF-DETR graph has no FLOAT casts to retarget") + return count + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/export_m48n_rf_detr_native_worker.py b/experiments/perception/export_m48n_rf_detr_native_worker.py new file mode 100644 index 0000000..4a663a0 --- /dev/null +++ b/experiments/perception/export_m48n_rf_detr_native_worker.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Export RF-DETR-L at the native KB4 canvas without image resampling.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Final + +import onnx # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48n-rf-detr-native-onnx-export/v0" +PROFILE_ID: Final = "rf-detr-large-coco-native-kb4-608x800-fp16/v0" +SOURCE_HEIGHT: Final = 600 +SOURCE_WIDTH: Final = 800 +MODEL_HEIGHT: Final = 608 +MODEL_WIDTH: Final = 800 +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--upstream-revision", required=True) + arguments = parser.parse_args() + + checkpoint = arguments.checkpoint.resolve(strict=True) + checkpoint_sha256 = sha256_path(checkpoint) + if checkpoint_sha256 != arguments.expected_checkpoint_sha256: + raise RuntimeError("checkpoint SHA-256 does not match the pinned finalist") + output_root = arguments.output_root.absolute() + manifest_path = arguments.manifest.absolute() + if output_root.exists() or manifest_path.exists(): + raise RuntimeError("native ONNX output or manifest already exists") + + from rfdetr import RFDETRLarge # type: ignore[import-not-found] + + started_utc_ns = time.time_ns() + model = RFDETRLarge(pretrain_weights=str(checkpoint)) + exported_path = Path( + model.export( + output_dir=str(output_root), + format="onnx", + shape=(MODEL_HEIGHT, MODEL_WIDTH), + batch_size=1, + dynamic_batch=False, + opset_version=17, + verbose=False, + notes={ + "missioncore_profile_id": PROFILE_ID, + "source_raster": [SOURCE_WIDTH, SOURCE_HEIGHT], + "model_canvas": [MODEL_WIDTH, MODEL_HEIGHT], + "geometric_resampling": False, + "padding": {"top": 0, "bottom": 8, "left": 0, "right": 0}, + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": checkpoint_sha256, + "authority": FALSE_AUTHORITY, + }, + ) + ).resolve(strict=True) + graph = onnx.load(str(exported_path), load_external_data=False) + onnx.checker.check_model(graph) + inputs = [_tensor_description(value) for value in graph.graph.input] + outputs = [_tensor_description(value) for value in graph.graph.output] + expected_input = [ + {"name": "input", "element_type": 1, "shape": [1, 3, MODEL_HEIGHT, MODEL_WIDTH]} + ] + if inputs != expected_input: + raise RuntimeError(f"unexpected native RF-DETR ONNX input contract: {inputs}") + if [item["name"] for item in outputs] != ["dets", "labels"]: + raise RuntimeError(f"unexpected native RF-DETR ONNX outputs: {outputs}") + + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": PROFILE_ID, + "provider_id": "shadow-rf-detr-large-coco-native-kb4-onnx/v0", + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": checkpoint_sha256, + "geometry": { + "source_raster_wh": [SOURCE_WIDTH, SOURCE_HEIGHT], + "model_canvas_wh": [MODEL_WIDTH, MODEL_HEIGHT], + "padding_tblr": [0, 8, 0, 0], + "resized": False, + "rectified": False, + "warped": False, + }, + "onnx": { + "path": str(exported_path), + "sha256": sha256_path(exported_path), + "size_bytes": exported_path.stat().st_size, + "opset_imports": [ + {"domain": item.domain, "version": item.version} for item in graph.opset_import + ], + "inputs": inputs, + "outputs": outputs, + }, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": FALSE_AUTHORITY, + } + manifest_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest_path.write_bytes(canonical_json(document) + b"\n") + print(exported_path) + print(json.dumps(document["onnx"], indent=2, sort_keys=True)) + return 0 + + +def _tensor_description(value: Any) -> dict[str, object]: + tensor = value.type.tensor_type + return { + "name": value.name, + "element_type": tensor.elem_type, + "shape": [_dimension_value(item) for item in tensor.shape.dim], + } + + +def _dimension_value(value: Any) -> int | str | None: + if value.HasField("dim_value"): + return int(value.dim_value) + if value.HasField("dim_param"): + return str(value.dim_param) + return None + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48n_native_parity_worker.py b/experiments/perception/run_m48n_native_parity_worker.py new file mode 100644 index 0000000..e701a7c --- /dev/null +++ b/experiments/perception/run_m48n_native_parity_worker.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Compare the native UINT8 TensorRT graph against the same RF-DETR PyTorch core.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import math +import statistics +import time +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final +from urllib.parse import urlsplit + +import av # type: ignore[import-not-found] +import numpy as np +from PIL import Image +from tritonclient import http as triton_http # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48n-native-pytorch-tensorrt-parity/v0" +SOURCE_HEIGHT: Final = 600 +SOURCE_WIDTH: Final = 800 +MODEL_HEIGHT: Final = 608 +MODEL_WIDTH: Final = 800 +FILL_VALUE: Final = 114 +MEANS: Final = (0.485, 0.456, 0.406) +STDS: Final = (0.229, 0.224, 0.225) +DEFAULT_FRAME_INDICES: Final = ( + 0, + 120, + 130, + 252, + 274, + 442, + 462, + 1093, + 1227, + 1453, + 1855, + 2385, + 2999, + 3999, + 4488, +) +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} +RISK_SPARSE_CLASS_IDS: Final = frozenset( + (1, 2, 3, 4, 6, 8, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 41) +) + + +@dataclass(frozen=True, slots=True) +class DecodedDetection: + sparse_class_id: int + score: float + box_xyxy: tuple[float, float, float, float] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--video", type=Path, required=True) + parser.add_argument("--expected-video-sha256", required=True) + parser.add_argument("--mask", type=Path, required=True) + parser.add_argument("--expected-mask-sha256", required=True) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--engine-sha256", required=True) + parser.add_argument("--triton-origin", default="http://127.0.0.1:8000") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--frame-indices", default=",".join(map(str, DEFAULT_FRAME_INDICES))) + arguments = parser.parse_args() + + for path, expected, label in ( + (arguments.video, arguments.expected_video_sha256, "video"), + (arguments.mask, arguments.expected_mask_sha256, "valid-FOV mask"), + (arguments.checkpoint, arguments.expected_checkpoint_sha256, "checkpoint"), + ): + if sha256_path(path.resolve(strict=True)) != expected: + raise RuntimeError(f"{label} SHA-256 changed") + output = arguments.output.absolute() + if output.exists(): + raise RuntimeError("native parity output already exists") + output.parent.mkdir(parents=True, exist_ok=True) + frame_indices = _frame_indices(arguments.frame_indices) + frames = load_frames(arguments.video, frame_indices) + mask = np.asarray(Image.open(arguments.mask).convert("L")) > 0 + if mask.shape != (SOURCE_HEIGHT, SOURCE_WIDTH) or not np.any(mask): + raise RuntimeError("valid-FOV mask geometry changed") + + import torch # type: ignore[import-not-found] + from rfdetr import RFDETRLarge # type: ignore[import-not-found] + from rfdetr.models.backbone.dinov2 import DinoV2 # type: ignore[import-not-found] + + model = RFDETRLarge(pretrain_weights=str(arguments.checkpoint)) + frozen_backbones = 0 + for module in model.model.model.modules(): + if isinstance(module, DinoV2): + module.shape = (MODEL_HEIGHT, MODEL_WIDTH) + module.export() + frozen_backbones += 1 + if frozen_backbones == 0: + raise RuntimeError("RF-DETR has no DINOv2 backbone to freeze for native shape") + model.inference(compile=False, dtype=torch.float16, inplace=True) + inference_model = model.model.inference_model + if inference_model is None: + raise RuntimeError("RF-DETR PyTorch inference model is unavailable") + endpoint = urlsplit(arguments.triton_origin) + if endpoint.scheme != "http" or not endpoint.hostname or endpoint.path not in ("", "/"): + raise RuntimeError("Triton origin must be an HTTP origin") + client = triton_http.InferenceServerClient( + url=f"{endpoint.hostname}:{endpoint.port or 80}", + verbose=False, + ) + if not client.is_model_ready("rf_detr_large_native_kb4", "1"): + raise RuntimeError("native Triton model is not ready") + + frame_rows: list[dict[str, object]] = [] + triton_ms: list[float] = [] + preprocessing_ms: list[float] = [] + boxes_absolute_errors: list[float] = [] + logits_absolute_errors: list[float] = [] + top50_jaccards: list[float] = [] + reference_risk_count = 0 + candidate_risk_count = 0 + matched_risk_count = 0 + matched_risk_ious: list[float] = [] + matched_risk_score_errors: list[float] = [] + started_utc_ns = time.time_ns() + try: + for frame_index in frame_indices: + image_bgr = frames[frame_index] + preprocess_started = time.perf_counter_ns() + reference_tensor = native_reference_preprocess(image_bgr, mask) + preprocessing_ms.append((time.perf_counter_ns() - preprocess_started) / 1_000_000) + torch_input = torch.from_numpy(reference_tensor).to("cuda", non_blocking=False) + torch.cuda.synchronize() + with torch.inference_mode(): + torch_output = inference_model(torch_input) + torch.cuda.synchronize() + pytorch_boxes, pytorch_logits = _pytorch_output(torch_output) + + triton_input = triton_http.InferInput( + "raw_kb4_bgr", + [1, SOURCE_HEIGHT, SOURCE_WIDTH, 3], + "UINT8", + ) + triton_input.set_data_from_numpy(image_bgr[None], binary_data=True) + requested = [ + triton_http.InferRequestedOutput("dets", binary_data=True), + triton_http.InferRequestedOutput("labels", binary_data=True), + ] + inference_started = time.perf_counter_ns() + response = client.infer( + "rf_detr_large_native_kb4", + [triton_input], + model_version="1", + outputs=requested, + ) + elapsed_ms = (time.perf_counter_ns() - inference_started) / 1_000_000 + triton_ms.append(elapsed_ms) + tensorrt_boxes = _array(response.as_numpy("dets"), (1, 300, 4), "dets") + tensorrt_logits = _array(response.as_numpy("labels"), (1, 300, 91), "labels") + + box_errors = np.abs( + pytorch_boxes.astype(np.float32) - tensorrt_boxes.astype(np.float32) + ) + logit_errors = np.abs( + pytorch_logits.astype(np.float32) - tensorrt_logits.astype(np.float32) + ) + boxes_absolute_errors.extend(float(value) for value in box_errors.reshape(-1)) + logits_absolute_errors.extend(float(value) for value in logit_errors.reshape(-1)) + pytorch_top50 = set( + int(value) + for value in np.argsort(-pytorch_logits[0].astype(np.float32).reshape(-1))[:50] + ) + tensorrt_top50 = set( + int(value) + for value in np.argsort(-tensorrt_logits[0].astype(np.float32).reshape(-1))[:50] + ) + top50_jaccard = len(pytorch_top50 & tensorrt_top50) / len( + pytorch_top50 | tensorrt_top50 + ) + top50_jaccards.append(top50_jaccard) + reference_risk = decode_risk_detections(pytorch_boxes, pytorch_logits) + candidate_risk = decode_risk_detections(tensorrt_boxes, tensorrt_logits) + matched = match_detections(reference_risk, candidate_risk) + reference_risk_count += len(reference_risk) + candidate_risk_count += len(candidate_risk) + matched_risk_count += len(matched) + matched_risk_ious.extend(item[0] for item in matched) + matched_risk_score_errors.extend(item[1] for item in matched) + frame_rows.append( + { + "frame_index": frame_index, + "triton_ms": round(elapsed_ms, 6), + "boxes_max_abs": round(float(box_errors.max()), 9), + "boxes_p99_abs": round(_percentile_array(box_errors, 0.99), 9), + "logits_max_abs": round(float(logit_errors.max()), 9), + "logits_p99_abs": round(_percentile_array(logit_errors, 0.99), 9), + "top50_query_class_jaccard": round(top50_jaccard, 9), + "reference_risk_detections": len(reference_risk), + "candidate_risk_detections": len(candidate_risk), + "matched_risk_detections_iou_at_least_0_5": len(matched), + } + ) + finally: + client.close() + + boxes_report = _error_report(boxes_absolute_errors) + logits_report = _error_report(logits_absolute_errors) + timing = _timing_report(triton_ms) + preprocessing_reference = _timing_report(preprocessing_ms) + risk_recall = matched_risk_count / reference_risk_count if reference_risk_count else 1.0 + risk_precision = matched_risk_count / candidate_risk_count if candidate_risk_count else 1.0 + risk_mean_iou = statistics.fmean(matched_risk_ious) if matched_risk_ious else 1.0 + risk_score_p95 = ( + _percentile(sorted(matched_risk_score_errors), 0.95) + if matched_risk_score_errors + else 0.0 + ) + gates = { + "risk_detection_recall_at_least_0_95": risk_recall >= 0.95, + "risk_detection_precision_at_least_0_95": risk_precision >= 0.95, + "matched_risk_mean_iou_at_least_0_90": risk_mean_iou >= 0.90, + "matched_risk_score_error_p95_at_most_0_10": risk_score_p95 <= 0.10, + "all_outputs_finite": all( + math.isfinite(value) for value in boxes_absolute_errors + logits_absolute_errors + ), + } + report: dict[str, object] = { + "schema_version": SCHEMA_VERSION, + "completed": True, + "passed": all(gates.values()), + "source": { + "video_sha256": arguments.expected_video_sha256, + "valid_fov_mask_sha256": arguments.expected_mask_sha256, + "frame_indices": list(frame_indices), + "raw_raster_wh": [SOURCE_WIDTH, SOURCE_HEIGHT], + }, + "candidate": { + "checkpoint_sha256": arguments.expected_checkpoint_sha256, + "engine_sha256": arguments.engine_sha256, + "input": { + "name": "raw_kb4_bgr", + "datatype": "UINT8", + "shape": [1, SOURCE_HEIGHT, SOURCE_WIDTH, 3], + "bytes_per_frame": SOURCE_HEIGHT * SOURCE_WIDTH * 3, + }, + "model_canvas_wh": [MODEL_WIDTH, MODEL_HEIGHT], + "pytorch_reference_frozen_dinov2_backbones": frozen_backbones, + "geometric_resampling": False, + "padding_tblr": [0, 8, 0, 0], + }, + "numeric_parity": { + "raw_query_diagnostics_not_acceptance_gates": { + "reason": "DETR top-k query identity is not stable across equivalent FP16 runtimes", + "boxes_absolute_error_by_query_index": boxes_report, + "logits_absolute_error_by_query_index": logits_report, + "mean_top50_query_class_jaccard": round(statistics.fmean(top50_jaccards), 9), + }, + "risk_semantic_output_parity": { + "minimum_score": 0.25, + "minimum_match_iou": 0.5, + "reference_detection_count": reference_risk_count, + "candidate_detection_count": candidate_risk_count, + "matched_detection_count": matched_risk_count, + "recall": round(risk_recall, 9), + "precision": round(risk_precision, 9), + "matched_mean_iou": round(risk_mean_iou, 9), + "matched_score_absolute_error_p95": round(risk_score_p95, 9), + }, + }, + "timing": { + "native_triton_raw_transport_and_inference_ms": timing, + "cpu_reference_preprocess_ms_not_in_candidate_path": preprocessing_reference, + }, + "frames": frame_rows, + "gates": gates, + "execution": { + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "packages": _package_versions( + ("rfdetr", "torch", "numpy", "av", "tritonclient") + ), + }, + "authority": FALSE_AUTHORITY, + } + report["report_identity_sha256"] = hashlib.sha256(canonical_json(report)).hexdigest() + output.write_bytes(canonical_json(report) + b"\n") + print(json.dumps({"output": str(output), "passed": report["passed"], "timing": timing})) + return 0 + + +def native_reference_preprocess( + image_bgr: np.ndarray[Any, np.dtype[np.uint8]], + mask: np.ndarray[Any, np.dtype[np.bool_]], +) -> np.ndarray[Any, np.dtype[np.float16]]: + if image_bgr.shape != (SOURCE_HEIGHT, SOURCE_WIDTH, 3) or image_bgr.dtype != np.uint8: + raise RuntimeError("raw KB4 frame contract changed") + raw = image_bgr.astype(np.float16) + valid = mask.astype(np.float16)[..., None] + invalid_fill = (~mask).astype(np.float16)[..., None] * np.float16(FILL_VALUE) + masked = raw * valid + invalid_fill + padded = np.full((MODEL_HEIGHT, MODEL_WIDTH, 3), FILL_VALUE, dtype=np.float16) + padded[:SOURCE_HEIGHT] = masked + nchw = np.ascontiguousarray(padded[:, :, ::-1].transpose(2, 0, 1))[None] + scale = np.asarray([1.0 / (255.0 * value) for value in STDS], dtype=np.float16) + bias = np.asarray( + [-mean / std for mean, std in zip(MEANS, STDS, strict=True)], + dtype=np.float16, + ) + return np.ascontiguousarray( + nchw * scale.reshape(1, 3, 1, 1) + bias.reshape(1, 3, 1, 1), + dtype=np.float16, + ) + + +def decode_risk_detections( + boxes: np.ndarray[Any, Any], + logits: np.ndarray[Any, Any], +) -> tuple[DecodedDetection, ...]: + probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits[0].astype(np.float32), -80, 80))) + flattened = probabilities.reshape(-1) + topk = np.argsort(-flattened, kind="stable")[:300] + result: list[DecodedDetection] = [] + for flat_index in topk: + score = float(flattened[flat_index]) + if score <= 0.25: + continue + sparse_class_id = int(flat_index % logits.shape[2]) + if sparse_class_id not in RISK_SPARSE_CLASS_IDS: + continue + query_index = int(flat_index // logits.shape[2]) + center_x, center_y, width, height = ( + float(value) for value in boxes[0, query_index].astype(np.float32) + ) + box = ( + min(max((center_x - width / 2) * MODEL_WIDTH, 0.0), float(SOURCE_WIDTH)), + min(max((center_y - height / 2) * MODEL_HEIGHT, 0.0), float(SOURCE_HEIGHT)), + min(max((center_x + width / 2) * MODEL_WIDTH, 0.0), float(SOURCE_WIDTH)), + min(max((center_y + height / 2) * MODEL_HEIGHT, 0.0), float(SOURCE_HEIGHT)), + ) + if box[2] <= box[0] or box[3] <= box[1]: + continue + result.append(DecodedDetection(sparse_class_id, score, box)) + return tuple(result) + + +def match_detections( + reference: tuple[DecodedDetection, ...], + candidate: tuple[DecodedDetection, ...], +) -> tuple[tuple[float, float], ...]: + unused = set(range(len(candidate))) + matches: list[tuple[float, float]] = [] + for expected in sorted(reference, key=lambda item: -item.score): + ranked = sorted( + ( + (_box_iou(expected.box_xyxy, candidate[index].box_xyxy), index) + for index in unused + if candidate[index].sparse_class_id == expected.sparse_class_id + ), + reverse=True, + ) + if not ranked or ranked[0][0] < 0.5: + continue + iou, index = ranked[0] + unused.remove(index) + matches.append((iou, abs(expected.score - candidate[index].score))) + return tuple(matches) + + +def _box_iou( + left: tuple[float, float, float, float], + right: tuple[float, float, float, float], +) -> float: + intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0])) + intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1])) + intersection = intersection_width * intersection_height + left_area = (left[2] - left[0]) * (left[3] - left[1]) + right_area = (right[2] - right[0]) * (right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0 else 0.0 + + +def load_frames( + video_path: Path, + indices: tuple[int, ...], +) -> dict[int, np.ndarray[Any, np.dtype[np.uint8]]]: + wanted = set(indices) + frames: dict[int, np.ndarray[Any, np.dtype[np.uint8]]] = {} + with av.open(str(video_path)) as container: + for index, frame in enumerate(container.decode(video=0)): + if index in wanted: + image = np.ascontiguousarray(frame.to_ndarray(format="bgr24"), dtype=np.uint8) + if image.shape != (SOURCE_HEIGHT, SOURCE_WIDTH, 3): + raise RuntimeError("decoded KB4 raster changed") + frames[index] = image + if len(frames) == len(indices): + break + missing = wanted - set(frames) + if missing: + raise RuntimeError(f"selected video frames are missing: {sorted(missing)}") + return frames + + +def _pytorch_output(value: object) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]: + if isinstance(value, Mapping): + boxes_value = value.get("pred_boxes") + logits_value = value.get("pred_logits") + elif isinstance(value, (tuple, list)) and len(value) == 2: + boxes_value, logits_value = value + else: + raise RuntimeError("PyTorch RF-DETR output contract changed") + for item in (boxes_value, logits_value): + if not hasattr(item, "detach"): + raise RuntimeError("PyTorch RF-DETR output is not a tensor") + boxes = boxes_value.detach().to("cpu").numpy() # type: ignore[union-attr] + logits = logits_value.detach().to("cpu").numpy() # type: ignore[union-attr] + return _array(boxes, (1, 300, 4), "PyTorch boxes"), _array( + logits, (1, 300, 91), "PyTorch logits" + ) + + +def _array(value: object, shape: tuple[int, ...], label: str) -> np.ndarray[Any, Any]: + if not isinstance(value, np.ndarray) or value.shape != shape or value.dtype != np.float16: + raise RuntimeError(f"{label} tensor contract changed") + if not np.isfinite(value).all(): + raise RuntimeError(f"{label} contains non-finite values") + return value + + +def _frame_indices(raw: str) -> tuple[int, ...]: + try: + result = tuple(sorted({int(value) for value in raw.split(",")})) + except ValueError as exc: + raise RuntimeError("frame indices are invalid") from exc + if not result or result[0] < 0: + raise RuntimeError("frame indices must be nonnegative") + return result + + +def _error_report(values: list[float]) -> dict[str, float]: + ordered = sorted(values) + return { + "mean": round(statistics.fmean(ordered), 9), + "p50": round(_percentile(ordered, 0.5), 9), + "p95": round(_percentile(ordered, 0.95), 9), + "p99": round(_percentile(ordered, 0.99), 9), + "maximum": round(ordered[-1], 9), + } + + +def _timing_report(values: list[float]) -> dict[str, float]: + ordered = sorted(values) + return { + "mean": round(statistics.fmean(ordered), 6), + "p50": round(_percentile(ordered, 0.5), 6), + "p95": round(_percentile(ordered, 0.95), 6), + "p99": round(_percentile(ordered, 0.99), 6), + "maximum": round(ordered[-1], 6), + } + + +def _percentile_array(values: np.ndarray[Any, Any], quantile: float) -> float: + return float(np.quantile(values.astype(np.float64), quantile)) + + +def _percentile(values: list[float], quantile: float) -> float: + position = (len(values) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return values[lower] + return values[lower] + (values[upper] - values[lower]) * (position - lower) + + +def _package_versions(names: tuple[str, ...]) -> dict[str, str]: + return {name: importlib.metadata.version(name) for name in names} + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48n_native_vs_704_worker.py b/experiments/perception/run_m48n_native_vs_704_worker.py new file mode 100644 index 0000000..5b0c7e1 --- /dev/null +++ b/experiments/perception/run_m48n_native_vs_704_worker.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""Compare native raw-KB4 RF-DETR against the frozen 704 pipeline on RAVNOVES00.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import statistics +import time +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final +from urllib.parse import urlsplit + +import av # type: ignore[import-not-found] +import cv2 # type: ignore[import-not-found] +import numpy as np +from PIL import Image +from tritonclient import http as triton_http # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48n-native-vs-704-ravnoves00/v0" +SOURCE_HEIGHT: Final = 600 +SOURCE_WIDTH: Final = 800 +NATIVE_MODEL_HEIGHT: Final = 608 +NATIVE_MODEL_WIDTH: Final = 800 +FILL_VALUE: Final = 114 +MEANS: Final = np.asarray((0.485, 0.456, 0.406), dtype=np.float32) +STDS: Final = np.asarray((0.229, 0.224, 0.225), dtype=np.float32) +RISK_LABEL_BY_SPARSE_ID: Final = { + 1: "person", + 2: "bicycle", + 3: "car", + 4: "motorcycle", + 6: "bus", + 8: "truck", + 16: "bird", + 17: "cat", + 18: "dog", + 19: "horse", + 20: "sheep", + 21: "cow", + 22: "elephant", + 23: "bear", + 24: "zebra", + 25: "giraffe", + 41: "skateboard", +} +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +@dataclass(frozen=True, slots=True) +class Detection: + label: str + score: float + box_xyxy: tuple[float, float, float, float] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--video", type=Path, required=True) + parser.add_argument("--expected-video-sha256", required=True) + parser.add_argument("--mask", type=Path, required=True) + parser.add_argument("--expected-mask-sha256", required=True) + parser.add_argument("--baseline-triton-origin", required=True) + parser.add_argument("--native-triton-origin", required=True) + parser.add_argument("--baseline-engine-sha256", required=True) + parser.add_argument("--native-engine-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--frames", type=Path, required=True) + parser.add_argument("--maximum-frames", type=int, default=0) + arguments = parser.parse_args() + + if arguments.maximum_frames < 0: + raise RuntimeError("maximum frames cannot be negative") + for path, expected, label in ( + (arguments.video, arguments.expected_video_sha256, "video"), + (arguments.mask, arguments.expected_mask_sha256, "valid-FOV mask"), + ): + if sha256_path(path.resolve(strict=True)) != expected: + raise RuntimeError(f"{label} SHA-256 changed") + for target in (arguments.output, arguments.frames): + if target.exists(): + raise RuntimeError(f"output already exists: {target}") + target.parent.mkdir(parents=True, exist_ok=True) + + mask = np.asarray(Image.open(arguments.mask).convert("L")) > 0 + if mask.shape != (SOURCE_HEIGHT, SOURCE_WIDTH) or not np.any(mask): + raise RuntimeError("valid-FOV mask geometry changed") + baseline_client = _client(arguments.baseline_triton_origin) + native_client = _client(arguments.native_triton_origin) + if not baseline_client.is_model_ready("rf_detr_large", "1"): + raise RuntimeError("frozen 704 Triton model is not ready") + if not native_client.is_model_ready("rf_detr_large_native_kb4", "1"): + raise RuntimeError("native Triton model is not ready") + + baseline_timing: dict[str, list[float]] = { + "preprocess": [], + "inference_transport": [], + "postprocess": [], + "total": [], + } + native_timing: dict[str, list[float]] = { + "client_prepare": [], + "inference_transport": [], + "postprocess": [], + "total": [], + } + baseline_rejected: Counter[str] = Counter() + native_rejected: Counter[str] = Counter() + baseline_labels: Counter[str] = Counter() + native_labels: Counter[str] = Counter() + baseline_detection_count = 0 + native_detection_count = 0 + matched_detection_count = 0 + matched_ious: list[float] = [] + matched_score_errors: list[float] = [] + started_utc_ns = time.time_ns() + frame_count = 0 + + zero = np.zeros((SOURCE_HEIGHT, SOURCE_WIDTH, 3), dtype=np.uint8) + _infer_baseline(baseline_client, preprocess_704(zero, mask)) + _infer_native(native_client, zero) + try: + with av.open(str(arguments.video)) as container, arguments.frames.open( + "x", encoding="utf-8" + ) as frame_stream: + for frame_index, frame in enumerate(container.decode(video=0)): + if arguments.maximum_frames and frame_index >= arguments.maximum_frames: + break + image_bgr = np.ascontiguousarray(frame.to_ndarray(format="bgr24"), dtype=np.uint8) + if image_bgr.shape != (SOURCE_HEIGHT, SOURCE_WIDTH, 3): + raise RuntimeError("decoded KB4 raster changed") + + baseline_started = time.perf_counter_ns() + baseline_preprocess_started = baseline_started + baseline_tensor = preprocess_704(image_bgr, mask) + baseline_inference_started = time.perf_counter_ns() + baseline_output = _infer_baseline(baseline_client, baseline_tensor) + baseline_postprocess_started = time.perf_counter_ns() + baseline_detections, rejected = postprocess( + *baseline_output, + mask=mask, + canvas_width=SOURCE_WIDTH, + canvas_height=SOURCE_HEIGHT, + ) + baseline_completed = time.perf_counter_ns() + baseline_rejected.update(rejected) + _append_timing( + baseline_timing, + baseline_preprocess_started, + baseline_inference_started, + baseline_postprocess_started, + baseline_completed, + first_key="preprocess", + ) + + native_started = time.perf_counter_ns() + native_input = np.ascontiguousarray(image_bgr, dtype=np.uint8) + native_inference_started = time.perf_counter_ns() + native_output = _infer_native(native_client, native_input) + native_postprocess_started = time.perf_counter_ns() + native_detections, rejected = postprocess( + *native_output, + mask=mask, + canvas_width=NATIVE_MODEL_WIDTH, + canvas_height=NATIVE_MODEL_HEIGHT, + ) + native_completed = time.perf_counter_ns() + native_rejected.update(rejected) + _append_timing( + native_timing, + native_started, + native_inference_started, + native_postprocess_started, + native_completed, + first_key="client_prepare", + ) + + matches = match_detections(baseline_detections, native_detections) + baseline_detection_count += len(baseline_detections) + native_detection_count += len(native_detections) + matched_detection_count += len(matches) + matched_ious.extend(item[0] for item in matches) + matched_score_errors.extend(item[1] for item in matches) + baseline_labels.update(item.label for item in baseline_detections) + native_labels.update(item.label for item in native_detections) + frame_stream.write( + canonical_json_text( + { + "frame_index": frame_index, + "baseline_detection_count": len(baseline_detections), + "native_detection_count": len(native_detections), + "matched_detection_count_iou_at_least_0_5": len(matches), + "baseline_total_ms": round( + (baseline_completed - baseline_started) / 1_000_000, 6 + ), + "native_total_ms": round( + (native_completed - native_started) / 1_000_000, 6 + ), + } + ) + + "\n" + ) + frame_count += 1 + if frame_count % 100 == 0: + frame_stream.flush() + print( + canonical_json_text( + { + "frame_count": frame_count, + "baseline_total_ms_latest": round( + baseline_timing["total"][-1], 6 + ), + "native_total_ms_latest": round( + native_timing["total"][-1], 6 + ), + } + ), + flush=True, + ) + finally: + baseline_client.close() + native_client.close() + + if frame_count == 0: + raise RuntimeError("RAVNOVES00 selection is empty") + baseline_recall = ( + matched_detection_count / baseline_detection_count if baseline_detection_count else 1.0 + ) + agreement_precision = ( + matched_detection_count / native_detection_count if native_detection_count else 1.0 + ) + baseline_total = _distribution(baseline_timing["total"]) + native_total = _distribution(native_timing["total"]) + checks = { + "native_total_p95_below_baseline": native_total["p95"] < baseline_total["p95"], + "native_total_mean_below_baseline": native_total["mean"] < baseline_total["mean"], + "native_total_p95_at_most_25_ms": native_total["p95"] <= 25.0, + "baseline_detection_retention_recall_at_least_0_80": baseline_recall >= 0.80, + "authority_remains_false": True, + } + report: dict[str, object] = { + "schema_version": SCHEMA_VERSION, + "completed": True, + "source": { + "source_id": "RAVNOVES00", + "video_sha256": arguments.expected_video_sha256, + "valid_fov_mask_sha256": arguments.expected_mask_sha256, + "frame_count": frame_count, + "full_video": arguments.maximum_frames == 0, + }, + "providers": { + "baseline_704": { + "engine_sha256": arguments.baseline_engine_sha256, + "preprocessing": "CPU mask+BGR-to-RGB+bilinear-704x704+ImageNet-normalize", + "input_bytes_per_frame": 1 * 3 * 704 * 704 * 4, + "detection_count": baseline_detection_count, + "label_counts": dict(sorted(baseline_labels.items())), + "rejected": dict(sorted(baseline_rejected.items())), + "timing_ms": {key: _distribution(value) for key, value in baseline_timing.items()}, + }, + "native_608x800": { + "engine_sha256": arguments.native_engine_sha256, + "preprocessing": "single TensorRT GPU graph; no geometric resampling", + "input_bytes_per_frame": SOURCE_HEIGHT * SOURCE_WIDTH * 3, + "detection_count": native_detection_count, + "label_counts": dict(sorted(native_labels.items())), + "rejected": dict(sorted(native_rejected.items())), + "timing_ms": {key: _distribution(value) for key, value in native_timing.items()}, + }, + }, + "comparison": { + "matched_detection_count_iou_at_least_0_5": matched_detection_count, + "baseline_detection_retention_recall": round(baseline_recall, 9), + "native_agreement_precision": round(agreement_precision, 9), + "matched_mean_iou": round(statistics.fmean(matched_ious), 9) + if matched_ious + else 1.0, + "matched_score_absolute_error_p95": round( + _percentile(sorted(matched_score_errors), 0.95), 9 + ) + if matched_score_errors + else 0.0, + "native_total_p95_delta_ms": round(native_total["p95"] - baseline_total["p95"], 6), + "native_total_mean_delta_ms": round(native_total["mean"] - baseline_total["mean"], 6), + "transport_bytes_reduction_fraction": round( + 1.0 - (SOURCE_HEIGHT * SOURCE_WIDTH * 3) / (1 * 3 * 704 * 704 * 4), + 9, + ), + }, + "checks": checks, + "execution": { + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "frames_sha256": sha256_path(arguments.frames), + }, + "authority": FALSE_AUTHORITY, + } + report["passed"] = all(checks.values()) + report["report_identity_sha256"] = hashlib.sha256(canonical_json(report)).hexdigest() + arguments.output.write_bytes(canonical_json(report) + b"\n") + print( + canonical_json_text( + { + "output": str(arguments.output), + "passed": report["passed"], + "frame_count": frame_count, + "baseline_total_ms": baseline_total, + "native_total_ms": native_total, + } + ), + flush=True, + ) + return 0 + + +def preprocess_704( + image_bgr: np.ndarray[Any, np.dtype[np.uint8]], + mask: np.ndarray[Any, np.dtype[np.bool_]], +) -> np.ndarray[Any, np.dtype[np.float32]]: + masked_bgr = np.where(mask[..., None], image_bgr, FILL_VALUE).astype(np.uint8) + rgb = np.ascontiguousarray(masked_bgr[:, :, ::-1]) + resized = cv2.resize(rgb, (704, 704), interpolation=cv2.INTER_LINEAR) + normalized = resized.astype(np.float32) / 255.0 + normalized = (normalized - MEANS) / STDS + return np.ascontiguousarray(normalized.transpose(2, 0, 1), dtype=np.float32)[None] + + +def postprocess( + boxes: np.ndarray[Any, Any], + logits: np.ndarray[Any, Any], + *, + mask: np.ndarray[Any, np.dtype[np.bool_]], + canvas_width: int, + canvas_height: int, +) -> tuple[tuple[Detection, ...], Counter[str]]: + if boxes.shape != (1, 300, 4) or boxes.dtype != np.float16: + raise RuntimeError("RF-DETR box tensor changed") + if logits.shape != (1, 300, 91) or logits.dtype != np.float16: + raise RuntimeError("RF-DETR logits tensor changed") + if not np.isfinite(boxes).all() or not np.isfinite(logits).all(): + raise RuntimeError("RF-DETR output contains non-finite values") + probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits[0].astype(np.float32), -80, 80))) + flattened = probabilities.reshape(-1) + topk = np.argsort(-flattened, kind="stable")[:300] + integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1) + result: list[Detection] = [] + rejected: Counter[str] = Counter() + for flat_index in topk: + score = float(flattened[flat_index]) + if score <= 0.25: + continue + sparse_class_id = int(flat_index % logits.shape[2]) + label = RISK_LABEL_BY_SPARSE_ID.get(sparse_class_id) + if label is None: + rejected["non-risk-or-unmapped-class"] += 1 + continue + query_index = int(flat_index // logits.shape[2]) + center_x, center_y, width, height = ( + float(value) for value in boxes[0, query_index].astype(np.float32) + ) + box = np.asarray( + ( + (center_x - width / 2) * canvas_width, + (center_y - height / 2) * canvas_height, + (center_x + width / 2) * canvas_width, + (center_y + height / 2) * canvas_height, + ), + dtype=np.float32, + ) + box[[0, 2]] = np.clip(box[[0, 2]], 0, SOURCE_WIDTH) + box[[1, 3]] = np.clip(box[[1, 3]], 0, SOURCE_HEIGHT) + fraction, center_inside, area = _valid_fraction(box, integral) + if area < 64.0: + rejected["small-box"] += 1 + continue + if area / (SOURCE_WIDTH * SOURCE_HEIGHT) > 0.5: + rejected["large-box"] += 1 + continue + if fraction < 0.5: + rejected["outside-valid-fov"] += 1 + continue + if not center_inside: + rejected["center-outside-valid-fov"] += 1 + continue + result.append( + Detection( + label, + score, + tuple(float(value) for value in box), # type: ignore[arg-type] + ) + ) + return tuple(sorted(result, key=lambda item: (-item.score, item.label))), rejected + + +def match_detections( + baseline: tuple[Detection, ...], + native: tuple[Detection, ...], +) -> tuple[tuple[float, float], ...]: + unused = set(range(len(native))) + matches: list[tuple[float, float]] = [] + for expected in baseline: + ranked = sorted( + ( + (_box_iou(expected.box_xyxy, native[index].box_xyxy), index) + for index in unused + if native[index].label == expected.label + ), + reverse=True, + ) + if not ranked or ranked[0][0] < 0.5: + continue + iou, index = ranked[0] + unused.remove(index) + matches.append((iou, abs(expected.score - native[index].score))) + return tuple(matches) + + +def _infer_baseline( + client: triton_http.InferenceServerClient, + tensor: np.ndarray[Any, np.dtype[np.float32]], +) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]: + return _infer( + client, + model_name="rf_detr_large", + input_name="input", + datatype="FP32", + tensor=tensor, + ) + + +def _infer_native( + client: triton_http.InferenceServerClient, + image_bgr: np.ndarray[Any, np.dtype[np.uint8]], +) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]: + tensor = np.ascontiguousarray(image_bgr[None], dtype=np.uint8) + return _infer( + client, + model_name="rf_detr_large_native_kb4", + input_name="raw_kb4_bgr", + datatype="UINT8", + tensor=tensor, + ) + + +def _infer( + client: triton_http.InferenceServerClient, + *, + model_name: str, + input_name: str, + datatype: str, + tensor: np.ndarray[Any, Any], +) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]: + request = triton_http.InferInput(input_name, list(tensor.shape), datatype) + request.set_data_from_numpy(tensor, binary_data=True) + response = client.infer( + model_name, + [request], + model_version="1", + outputs=[ + triton_http.InferRequestedOutput("dets", binary_data=True), + triton_http.InferRequestedOutput("labels", binary_data=True), + ], + ) + boxes = response.as_numpy("dets") + logits = response.as_numpy("labels") + if not isinstance(boxes, np.ndarray) or not isinstance(logits, np.ndarray): + raise RuntimeError("Triton RF-DETR output is missing") + return boxes, logits + + +def _client(origin: str) -> triton_http.InferenceServerClient: + endpoint = urlsplit(origin) + if endpoint.scheme != "http" or not endpoint.hostname or endpoint.path not in ("", "/"): + raise RuntimeError("Triton endpoint must be an HTTP origin") + return triton_http.InferenceServerClient( + url=f"{endpoint.hostname}:{endpoint.port or 80}", verbose=False + ) + + +def _append_timing( + destination: dict[str, list[float]], + started: int, + inference_started: int, + postprocess_started: int, + completed: int, + *, + first_key: str, +) -> None: + destination[first_key].append((inference_started - started) / 1_000_000) + destination["inference_transport"].append( + (postprocess_started - inference_started) / 1_000_000 + ) + destination["postprocess"].append((completed - postprocess_started) / 1_000_000) + destination["total"].append((completed - started) / 1_000_000) + + +def _valid_fraction( + box: np.ndarray[Any, np.dtype[np.float32]], + integral: np.ndarray[Any, np.dtype[np.int64]], +) -> tuple[float, bool, float]: + x1 = int(np.clip(math.floor(float(box[0])), 0, SOURCE_WIDTH)) + y1 = int(np.clip(math.floor(float(box[1])), 0, SOURCE_HEIGHT)) + x2 = int(np.clip(math.ceil(float(box[2])), 0, SOURCE_WIDTH)) + y2 = int(np.clip(math.ceil(float(box[3])), 0, SOURCE_HEIGHT)) + area = float(max(0, x2 - x1) * max(0, y2 - y1)) + if area <= 0: + return 0.0, False, 0.0 + inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1] + center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2), 0, SOURCE_WIDTH - 1)) + center_y = int( + np.clip(round((float(box[1]) + float(box[3])) / 2), 0, SOURCE_HEIGHT - 1) + ) + center_inside = bool( + integral[center_y + 1, center_x + 1] + - integral[center_y, center_x + 1] + - integral[center_y + 1, center_x] + + integral[center_y, center_x] + ) + return float(inside) / area, center_inside, area + + +def _box_iou( + left: tuple[float, float, float, float], + right: tuple[float, float, float, float], +) -> float: + intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0])) + intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1])) + intersection = intersection_width * intersection_height + left_area = (left[2] - left[0]) * (left[3] - left[1]) + right_area = (right[2] - right[0]) * (right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0 else 0.0 + + +def _distribution(values: list[float]) -> dict[str, float]: + ordered = sorted(values) + return { + "mean": round(statistics.fmean(ordered), 6), + "p50": round(_percentile(ordered, 0.5), 6), + "p95": round(_percentile(ordered, 0.95), 6), + "p99": round(_percentile(ordered, 0.99), 6), + "maximum": round(ordered[-1], 6), + } + + +def _percentile(values: list[float], quantile: float) -> float: + position = (len(values) - 1) * quantile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return values[lower] + return values[lower] + (values[upper] - values[lower]) * (position - lower) + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return canonical_json_text(value).encode("utf-8") + + +def canonical_json_text(value: object) -> str: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/Invoke-M48NNativeCandidateBuild.ps1 b/experiments/perception/worker/Invoke-M48NNativeCandidateBuild.ps1 new file mode 100644 index 0000000..060a5a2 --- /dev/null +++ b/experiments/perception/worker/Invoke-M48NNativeCandidateBuild.ps1 @@ -0,0 +1,349 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ReleaseRoot, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9._-]{1,96}$")] + [string]$RunId, + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\experiments\m48n-native-candidate" +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { + if ($Create -and -not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Assert-RegularFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "$Label must be a regular file" + } + return $item.FullName +} + +function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } + +function Get-Container([string]$Name) { + $rows = @((& docker inspect $Name) | ConvertFrom-Json) + Assert-LastExitCode "Docker inspection for $Name" + if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" } + return $rows[0] +} + +if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { + throw "M48N native candidate build is pinned to DESKTOP-OPJ8J04" +} + +$release = Resolve-DDirectory $ReleaseRoot "M48N release root" $false +$output = Resolve-DDirectory $OutputRoot "M48N output root" $true +$runOutput = Join-Path $output $RunId +if (Test-Path -LiteralPath $runOutput) { throw "M48N run output already exists" } +$null = New-Item -ItemType Directory -Path $runOutput +$runOutput = Resolve-DDirectory $runOutput "M48N run output" $false + +$exporter = Assert-RegularFile ( + Join-Path $release "export_m48n_rf_detr_native_worker.py" +) "native exporter" +$converter = Assert-RegularFile ( + Join-Path $release "convert_m48n_rf_detr_native_onnx_fp16.py" +) "native FP16 converter" +$wrapper = Assert-RegularFile ( + Join-Path $release "wrap_m48n_rf_detr_native_uint8_onnx.py" +) "native UINT8 wrapper" +$checkpoint = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z" + + "\weights\rf-detr-large-2026.pth" +) "RF-DETR checkpoint" +$checkpointSha256 = Get-Sha256 $checkpoint +if ($checkpointSha256 -cne "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38") { + throw "RF-DETR checkpoint SHA-256 changed" +} +$mask = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\inputs\e2" + + "\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" + + "\mask.png" +) "valid-FOV mask" +$maskSha256 = Get-Sha256 $mask +if ($maskSha256 -cne "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63") { + throw "valid-FOV mask SHA-256 changed" +} + +$historical = Get-Container "ndc-mission-core-triton" +if (-not $historical.State.Running -or $historical.State.Health.Status -cne "healthy") { + throw "Historical Triton must remain healthy" +} +$historicalId = [string]$historical.Id +$baseImage = ( + "nvcr.io/nvidia/tritonserver:26.06-py3@" + + "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" +) +& docker image inspect $baseImage *> $null +Assert-LastExitCode "pinned base image inspection" +$baseImageId = (& docker image inspect $baseImage --format "{{.Id}}").Trim() +Assert-LastExitCode "pinned base image identity" +$runtimeVolume = "ndc-mission-core-m48t-upstream-parity-env" +if (-not (& docker volume ls --quiet --filter "name=^$runtimeVolume$")) { + throw "pinned RF-DETR dependency volume is unavailable" +} +$buildDepsVolume = "ndc-mission-core-m48n-native-build-deps" +if (-not (& docker volume ls --quiet --filter "name=^$buildDepsVolume$")) { + & docker volume create ` + --label "com.nodedc.product=mission-core" ` + --label "com.nodedc.stack=ndc-mission-core-compute" ` + --label "com.nodedc.role=bounded-rf-detr-native-build-dependencies" ` + --label "com.nodedc.managed-by=codex-bounded-experiment" ` + $buildDepsVolume *> $null + Assert-LastExitCode "M48N build dependency volume creation" +} +$buildDepsReady = $false +$strictErrorActionPreference = $ErrorActionPreference +$ErrorActionPreference = "Continue" +& docker run --rm ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --tmpfs "/tmp:rw,noexec,nosuid,size=512m" ` + -e "PYTHONPATH=/opt/build-deps:/opt/parity" ` + -v ($buildDepsVolume + ":/opt/build-deps:ro") ` + -v ($runtimeVolume + ":/opt/parity:ro") ` + --entrypoint python3 ` + $baseImage ` + -c ( + "import importlib.metadata as m, onnx, onnxconverter_common; " + + "assert m.version('onnx') == '1.22.0'; " + + "assert m.version('onnxconverter-common') == '1.16.0'; " + + "assert m.version('protobuf') == '6.33.6'" + ) *> $null +if ($LASTEXITCODE -eq 0) { $buildDepsReady = $true } +$ErrorActionPreference = $strictErrorActionPreference +if (-not $buildDepsReady) { + & docker run --rm ` + --name "ndc-mission-core-m48n-native-build-deps-init" ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --pids-limit 256 ` + --tmpfs "/tmp:rw,noexec,nosuid,size=2g" ` + -e "PIP_DISABLE_PIP_VERSION_CHECK=1" ` + -v ($buildDepsVolume + ":/opt/build-deps:rw") ` + --entrypoint python3 ` + $baseImage ` + -m pip install --no-cache-dir --no-deps --upgrade --target /opt/build-deps ` + "onnx==1.22.0" "onnxconverter-common==1.16.0" "protobuf==6.33.6" + Assert-LastExitCode "M48N build dependency initialization" +} +& docker run --rm ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --tmpfs "/tmp:rw,noexec,nosuid,size=512m" ` + -e "PYTHONPATH=/opt/build-deps:/opt/parity" ` + -v ($buildDepsVolume + ":/opt/build-deps:ro") ` + -v ($runtimeVolume + ":/opt/parity:ro") ` + --entrypoint python3 ` + $baseImage ` + -c ( + "import importlib.metadata as m, onnx, onnxconverter_common; " + + "assert m.version('onnx') == '1.22.0'; " + + "assert m.version('onnxconverter-common') == '1.16.0'; " + + "assert m.version('protobuf') == '6.33.6'" + ) +Assert-LastExitCode "M48N build dependency verification" + +$releaseMount = (Convert-ToDockerPath $release) + ":/release:ro" +$outputMount = (Convert-ToDockerPath $runOutput) + ":/output:rw" +$checkpointMount = (Convert-ToDockerPath $checkpoint) + ":/model/rf-detr-large-2026.pth:ro" +$maskMount = (Convert-ToDockerPath $mask) + ":/input/mask.png:ro" +$common = @( + "run", "--rm", + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "512", + "--gpus", "all", + "--shm-size", "2g", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=8g", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", "PYTHONPATH=/opt/build-deps:/opt/parity", + "-v", ($buildDepsVolume + ":/opt/build-deps:ro"), + "-v", ($runtimeVolume + ":/opt/parity:ro"), + "-v", $releaseMount, + "-v", $outputMount, + "-v", $checkpointMount, + "-v", $maskMount +) + +try { + & docker @common ` + --name "ndc-mission-core-m48n-native-export" ` + --entrypoint python3 ` + $baseImage ` + /release/export_m48n_rf_detr_native_worker.py ` + --checkpoint /model/rf-detr-large-2026.pth ` + --expected-checkpoint-sha256 $checkpointSha256 ` + --output-root /output/core-export ` + --manifest /output/export-manifest.json ` + --upstream-revision 9b009fa928d6218320439803d1da01869a85c072 + Assert-LastExitCode "M48N native core export" + + $exportManifest = Get-Content -LiteralPath ( + Join-Path $runOutput "export-manifest.json" + ) -Raw | ConvertFrom-Json + $corePath = [string]$exportManifest.onnx.path + $coreLeaf = Split-Path -Leaf $corePath + $coreHostPath = Assert-RegularFile ( + Join-Path (Join-Path $runOutput "core-export") $coreLeaf + ) "native core ONNX" + if ((Get-Sha256 $coreHostPath) -cne [string]$exportManifest.onnx.sha256) { + throw "native core ONNX hash does not match its manifest" + } + + & docker @common ` + --name "ndc-mission-core-m48n-native-fp16" ` + --entrypoint python3 ` + $baseImage ` + /release/convert_m48n_rf_detr_native_onnx_fp16.py ` + --input ("/output/core-export/" + $coreLeaf) ` + --expected-input-sha256 ([string]$exportManifest.onnx.sha256) ` + --output /output/rf-detr-native-core-fp16.onnx ` + --manifest /output/fp16-manifest.json + Assert-LastExitCode "M48N native FP16 conversion" + + $fp16Manifest = Get-Content -LiteralPath ( + Join-Path $runOutput "fp16-manifest.json" + ) -Raw | ConvertFrom-Json + $fp16Path = Assert-RegularFile ( + Join-Path $runOutput "rf-detr-native-core-fp16.onnx" + ) "native FP16 ONNX" + if ((Get-Sha256 $fp16Path) -cne [string]$fp16Manifest.output_onnx_sha256) { + throw "native FP16 ONNX hash does not match its manifest" + } + + & docker @common ` + --name "ndc-mission-core-m48n-native-wrapper" ` + --entrypoint python3 ` + $baseImage ` + /release/wrap_m48n_rf_detr_native_uint8_onnx.py ` + --input /output/rf-detr-native-core-fp16.onnx ` + --expected-input-sha256 ([string]$fp16Manifest.output_onnx_sha256) ` + --mask /input/mask.png ` + --expected-mask-sha256 $maskSha256 ` + --output /output/rf-detr-native-uint8.onnx ` + --manifest /output/wrapper-manifest.json + Assert-LastExitCode "M48N native UINT8 wrapper" + + $wrapperManifest = Get-Content -LiteralPath ( + Join-Path $runOutput "wrapper-manifest.json" + ) -Raw | ConvertFrom-Json + $wrappedPath = Assert-RegularFile ( + Join-Path $runOutput "rf-detr-native-uint8.onnx" + ) "native UINT8 ONNX" + if ((Get-Sha256 $wrappedPath) -cne [string]$wrapperManifest.output_onnx_sha256) { + throw "native UINT8 ONNX hash does not match its manifest" + } + + & docker @common ` + --name "ndc-mission-core-m48n-native-trt-build" ` + --entrypoint /usr/src/tensorrt/bin/trtexec ` + $baseImage ` + --onnx=/output/rf-detr-native-uint8.onnx ` + --saveEngine=/output/rf-detr-native-uint8.plan ` + --stronglyTyped ` + --builderOptimizationLevel=5 ` + --memPoolSize=workspace:8192 ` + --skipInference + Assert-LastExitCode "M48N native TensorRT engine build" + + $enginePath = Assert-RegularFile ( + Join-Path $runOutput "rf-detr-native-uint8.plan" + ) "native TensorRT engine" + $receipt = [ordered]@{ + schema_version = "missioncore.m48n-native-candidate-build/v0" + run_id = $RunId + completed = $true + worker = [ordered]@{ + id = "worker-006" + node = $env:COMPUTERNAME + base_image = $baseImage + base_image_id = $baseImageId + dependency_volume = $runtimeVolume + build_dependency_volume = $buildDepsVolume + } + artifacts = [ordered]@{ + export_manifest_sha256 = Get-Sha256 ( + Join-Path $runOutput "export-manifest.json" + ) + core_onnx_sha256 = Get-Sha256 $coreHostPath + fp16_manifest_sha256 = Get-Sha256 ( + Join-Path $runOutput "fp16-manifest.json" + ) + fp16_onnx_sha256 = Get-Sha256 $fp16Path + wrapper_manifest_sha256 = Get-Sha256 ( + Join-Path $runOutput "wrapper-manifest.json" + ) + wrapped_onnx_sha256 = Get-Sha256 $wrappedPath + engine_sha256 = Get-Sha256 $enginePath + engine_size_bytes = (Get-Item -LiteralPath $enginePath).Length + } + historical_triton = [ordered]@{ + container_id = $historicalId + action = "none" + } + authority = [ordered]@{ + ground_truth = $false + candidate_accepted = $false + commands_enabled = $false + actuation_allowed = $false + navigation_or_safety_accepted = $false + } + } + $receipt | ConvertTo-Json -Depth 8 -Compress | Set-Content -LiteralPath ( + Join-Path $runOutput "build-receipt.json" + ) -Encoding UTF8 +} finally { + foreach ($name in @( + "ndc-mission-core-m48n-native-export", + "ndc-mission-core-m48n-native-fp16", + "ndc-mission-core-m48n-native-wrapper", + "ndc-mission-core-m48n-native-trt-build" + )) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + & docker rm -f $name *> $null + } + } + $historicalAfter = Get-Container "ndc-mission-core-triton" + if ( + $historicalAfter.Id -cne $historicalId -or + -not $historicalAfter.State.Running -or + $historicalAfter.State.Health.Status -cne "healthy" + ) { + throw "Historical Triton changed during M48N candidate build" + } +} + +Write-Output ("M48N_NATIVE_BUILD_RECEIPT={0}" -f ( + Join-Path $runOutput "build-receipt.json" +)) +Write-Output "HISTORICAL_TRITON_ACTION=none" +Write-Output "SEMANTIC_AUTHORITY_CHANGED=false" diff --git a/experiments/perception/worker/Invoke-M48NNativeParity.ps1 b/experiments/perception/worker/Invoke-M48NNativeParity.ps1 new file mode 100644 index 0000000..eff7b3a --- /dev/null +++ b/experiments/perception/worker/Invoke-M48NNativeParity.ps1 @@ -0,0 +1,238 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ReleaseRoot, + [Parameter(Mandatory = $true)] + [string]$CandidateRoot, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9._-]{1,96}$")] + [string]$RunId, + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48n-native-parity" +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { + if ($Create -and -not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Assert-RegularFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "$Label must be a regular file" + } + return $item.FullName +} + +function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } + +function Get-Container([string]$Name) { + $rows = @((& docker inspect $Name) | ConvertFrom-Json) + Assert-LastExitCode "Docker inspection for $Name" + if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" } + return $rows[0] +} + +if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { + throw "M48N native parity is pinned to DESKTOP-OPJ8J04" +} + +$release = Resolve-DDirectory $ReleaseRoot "M48N release root" $false +$candidate = Resolve-DDirectory $CandidateRoot "M48N candidate root" $false +$output = Resolve-DDirectory $OutputRoot "M48N parity output root" $true +$runOutput = Join-Path $output $RunId +if (Test-Path -LiteralPath $runOutput) { throw "M48N parity output already exists" } +$null = New-Item -ItemType Directory -Path $runOutput +$runOutput = Resolve-DDirectory $runOutput "M48N parity run output" $false + +$runner = Assert-RegularFile ( + Join-Path $release "run_m48n_native_parity_worker.py" +) "native parity runner" +$modelConfig = Assert-RegularFile ( + Join-Path $release "rf_detr_large_native_kb4_config.pbtxt" +) "native Triton config" +$engine = Assert-RegularFile ( + Join-Path $candidate "rf-detr-native-uint8.plan" +) "native TensorRT engine" +$engineSha256 = Get-Sha256 $engine +if ($engineSha256 -cne "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695") { + throw "native TensorRT engine SHA-256 changed" +} +$video = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" + + "\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4" +) "RAVNOVES00 video" +$videoSha256 = Get-Sha256 $video +if ($videoSha256 -cne "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8") { + throw "RAVNOVES00 video SHA-256 changed" +} +$mask = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\inputs\e2" + + "\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" + + "\mask.png" +) "valid-FOV mask" +$maskSha256 = Get-Sha256 $mask +if ($maskSha256 -cne "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63") { + throw "valid-FOV mask SHA-256 changed" +} +$checkpoint = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z" + + "\weights\rf-detr-large-2026.pth" +) "RF-DETR checkpoint" +$checkpointSha256 = Get-Sha256 $checkpoint +if ($checkpointSha256 -cne "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38") { + throw "RF-DETR checkpoint SHA-256 changed" +} + +$historical = Get-Container "ndc-mission-core-triton" +if (-not $historical.State.Running -or $historical.State.Health.Status -cne "healthy") { + throw "Historical Triton must remain healthy" +} +$historicalId = [string]$historical.Id +$baseImage = ( + "nvcr.io/nvidia/tritonserver:26.06-py3@" + + "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" +) +& docker image inspect $baseImage *> $null +Assert-LastExitCode "pinned base image inspection" +$runtimeVolume = "ndc-mission-core-m48t-upstream-parity-env" +if (-not (& docker volume ls --quiet --filter "name=^$runtimeVolume$")) { + throw "pinned RF-DETR dependency volume is unavailable" +} + +$modelRoot = Join-Path $runOutput "triton-models" +$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4" +$modelVersionDirectory = Join-Path $modelDirectory "1" +$null = New-Item -ItemType Directory -Path $modelVersionDirectory +Copy-Item -LiteralPath $modelConfig -Destination (Join-Path $modelDirectory "config.pbtxt") +Copy-Item -LiteralPath $engine -Destination (Join-Path $modelVersionDirectory "model.plan") +if ((Get-Sha256 (Join-Path $modelVersionDirectory "model.plan")) -cne $engineSha256) { + throw "staged native TensorRT engine SHA-256 changed" +} + +$tritonName = "ndc-mission-core-m48n-native-parity-triton" +$runnerName = "ndc-mission-core-m48n-native-parity" +foreach ($name in @($tritonName, $runnerName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + throw "M48N parity container $name already exists" + } +} + +try { + & docker create ` + --name $tritonName ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --pids-limit 512 ` + --shm-size 1g ` + --gpus all ` + --tmpfs "/tmp:rw,noexec,nosuid,size=2g" ` + --health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" ` + --health-interval 5s ` + --health-timeout 3s ` + --health-start-period 20s ` + --health-retries 24 ` + -v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") ` + $baseImage ` + tritonserver ` + --model-repository=/models ` + --model-control-mode=explicit ` + --load-model=rf_detr_large_native_kb4 ` + --disable-auto-complete-config ` + --strict-readiness=true ` + --exit-on-error=true ` + --allow-http=true ` + --allow-grpc=false ` + --allow-metrics=false *> $null + Assert-LastExitCode "M48N parity Triton creation" + & docker start $tritonName *> $null + Assert-LastExitCode "M48N parity Triton start" + $ready = $false + foreach ($attempt in 1..60) { + Start-Sleep -Seconds 2 + $candidateContainer = Get-Container $tritonName + if (-not $candidateContainer.State.Running) { + & docker logs $tritonName + throw "M48N parity Triton stopped during startup" + } + if ($candidateContainer.State.Health.Status -ceq "healthy") { + $ready = $true + break + } + } + if (-not $ready) { throw "M48N parity Triton did not become healthy" } + + & docker run ` + --name $runnerName ` + --network ("container:{0}" -f $tritonName) ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --pids-limit 512 ` + --gpus all ` + --shm-size 2g ` + --tmpfs "/tmp:rw,noexec,nosuid,size=8g" ` + -e "PYTHONDONTWRITEBYTECODE=1" ` + -e "PYTHONPATH=/opt/parity" ` + -v ($runtimeVolume + ":/opt/parity:ro") ` + -v ((Convert-ToDockerPath $release) + ":/release:ro") ` + -v ((Convert-ToDockerPath $video) + ":/input/video.mp4:ro") ` + -v ((Convert-ToDockerPath $mask) + ":/input/mask.png:ro") ` + -v ((Convert-ToDockerPath $checkpoint) + ":/model/rf-detr-large-2026.pth:ro") ` + -v ((Convert-ToDockerPath $runOutput) + ":/output:rw") ` + --entrypoint python3 ` + $baseImage ` + /release/run_m48n_native_parity_worker.py ` + --video /input/video.mp4 ` + --expected-video-sha256 $videoSha256 ` + --mask /input/mask.png ` + --expected-mask-sha256 $maskSha256 ` + --checkpoint /model/rf-detr-large-2026.pth ` + --expected-checkpoint-sha256 $checkpointSha256 ` + --engine-sha256 $engineSha256 ` + --triton-origin http://127.0.0.1:8000 ` + --output /output/result.json + Assert-LastExitCode "M48N native PyTorch/TensorRT parity" + if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) { + throw "M48N native parity result was not written" + } +} finally { + foreach ($name in @($runnerName, $tritonName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + & docker rm -f $name *> $null + } + } + $historicalAfter = Get-Container "ndc-mission-core-triton" + if ( + $historicalAfter.Id -cne $historicalId -or + -not $historicalAfter.State.Running -or + $historicalAfter.State.Health.Status -cne "healthy" + ) { + throw "Historical Triton changed during M48N parity" + } +} + +Write-Output ("M48N_NATIVE_PARITY_RESULT={0}" -f (Join-Path $runOutput "result.json")) +Write-Output "HISTORICAL_TRITON_ACTION=none" +Write-Output "SEMANTIC_AUTHORITY_CHANGED=false" diff --git a/experiments/perception/worker/Invoke-M48NNativeVs704.ps1 b/experiments/perception/worker/Invoke-M48NNativeVs704.ps1 new file mode 100644 index 0000000..89f33ee --- /dev/null +++ b/experiments/perception/worker/Invoke-M48NNativeVs704.ps1 @@ -0,0 +1,275 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ReleaseRoot, + [Parameter(Mandatory = $true)] + [string]$CandidateRoot, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9._-]{1,96}$")] + [string]$RunId, + [ValidateRange(0, 1000000)] + [int]$MaximumFrames = 0, + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\results\m48n-native-vs-704" +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { + if ($Create -and -not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Assert-RegularFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "$Label must be a regular file" + } + return $item.FullName +} + +function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } + +function Get-Container([string]$Name) { + $rows = @((& docker inspect $Name) | ConvertFrom-Json) + Assert-LastExitCode "Docker inspection for $Name" + if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" } + return $rows[0] +} + +if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { + throw "M48N native/704 comparison is pinned to DESKTOP-OPJ8J04" +} + +$release = Resolve-DDirectory $ReleaseRoot "M48N release root" $false +$candidate = Resolve-DDirectory $CandidateRoot "M48N candidate root" $false +$output = Resolve-DDirectory $OutputRoot "M48N comparison output root" $true +$runOutput = Join-Path $output $RunId +if (Test-Path -LiteralPath $runOutput) { throw "M48N comparison output already exists" } +$null = New-Item -ItemType Directory -Path $runOutput +$runOutput = Resolve-DDirectory $runOutput "M48N comparison run output" $false + +$runner = Assert-RegularFile ( + Join-Path $release "run_m48n_native_vs_704_worker.py" +) "native/704 comparison runner" +$nativeConfig = Assert-RegularFile ( + Join-Path $release "rf_detr_large_native_kb4_config.pbtxt" +) "native Triton config" +$nativeEngine = Assert-RegularFile ( + Join-Path $candidate "rf-detr-native-uint8.plan" +) "native TensorRT engine" +$nativeEngineSha256 = Get-Sha256 $nativeEngine +if ($nativeEngineSha256 -cne "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695") { + throw "native TensorRT engine SHA-256 changed" +} +$baselineRoot = Resolve-DDirectory ( + "D:\NDC_MISSIONCORE\runtime\experiments\m48t-fixed-detector-20260825T095425Z" + + "\triton-models\rf_detr_large" +) "frozen RF-DETR 704 model root" $false +$baselineConfig = Assert-RegularFile ( + Join-Path $baselineRoot "config.pbtxt" +) "frozen RF-DETR 704 Triton config" +if ((Get-Sha256 $baselineConfig) -cne "80947cad235e5b000f11aa869a33af0e8c727f07e04046691468df1e171479b6") { + throw "frozen RF-DETR 704 Triton config SHA-256 changed" +} +$baselineEngine = Assert-RegularFile ( + Join-Path $baselineRoot "1\model.plan" +) "frozen RF-DETR 704 TensorRT engine" +$baselineEngineSha256 = Get-Sha256 $baselineEngine +if ($baselineEngineSha256 -cne "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8") { + throw "frozen RF-DETR 704 TensorRT engine SHA-256 changed" +} +$video = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" + + "\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4" +) "RAVNOVES00 video" +$videoSha256 = Get-Sha256 $video +if ($videoSha256 -cne "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8") { + throw "RAVNOVES00 video SHA-256 changed" +} +$mask = Assert-RegularFile ( + "D:\NDC_MISSIONCORE\runtime\inputs\e2" + + "\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" + + "\mask.png" +) "valid-FOV mask" +$maskSha256 = Get-Sha256 $mask +if ($maskSha256 -cne "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63") { + throw "valid-FOV mask SHA-256 changed" +} +$opencvPackages = Resolve-DDirectory ( + "D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages" +) "pinned OpenCV packages" $false + +$historical = Get-Container "ndc-mission-core-triton" +if (-not $historical.State.Running -or $historical.State.Health.Status -cne "healthy") { + throw "Canonical Triton must remain healthy" +} +$historicalId = [string]$historical.Id +$baseImage = ( + "nvcr.io/nvidia/tritonserver:26.06-py3@" + + "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" +) +& docker image inspect $baseImage *> $null +Assert-LastExitCode "pinned base image inspection" +$runtimeVolume = "ndc-mission-core-m48t-upstream-parity-env" +if (-not (& docker volume ls --quiet --filter "name=^$runtimeVolume$")) { + throw "pinned RF-DETR dependency volume is unavailable" +} + +$modelRoot = Join-Path $runOutput "triton-models" +$baselineModelDirectory = Join-Path $modelRoot "rf_detr_large" +$baselineVersionDirectory = Join-Path $baselineModelDirectory "1" +$nativeModelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4" +$nativeVersionDirectory = Join-Path $nativeModelDirectory "1" +$null = New-Item -ItemType Directory -Path $baselineVersionDirectory +$null = New-Item -ItemType Directory -Path $nativeVersionDirectory +Copy-Item -LiteralPath $baselineConfig -Destination (Join-Path $baselineModelDirectory "config.pbtxt") +Copy-Item -LiteralPath $baselineEngine -Destination (Join-Path $baselineVersionDirectory "model.plan") +Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $nativeModelDirectory "config.pbtxt") +Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $nativeVersionDirectory "model.plan") +if ((Get-Sha256 (Join-Path $baselineVersionDirectory "model.plan")) -cne $baselineEngineSha256) { + throw "staged RF-DETR 704 TensorRT engine SHA-256 changed" +} +if ((Get-Sha256 (Join-Path $nativeVersionDirectory "model.plan")) -cne $nativeEngineSha256) { + throw "staged native TensorRT engine SHA-256 changed" +} + +$tritonName = "ndc-mission-core-m48n-native-vs-704-triton" +$runnerName = "ndc-mission-core-m48n-native-vs-704" +foreach ($name in @($tritonName, $runnerName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + throw "M48N comparison container $name already exists" + } +} + +try { + & docker create ` + --name $tritonName ` + --label "com.nodedc.product=mission-core" ` + --label "com.nodedc.stack=ndc-mission-core-compute" ` + --label "com.nodedc.role=bounded-rf-detr-native-vs-704" ` + --label "com.nodedc.managed-by=codex-bounded-experiment" ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --pids-limit 512 ` + --shm-size 1g ` + --gpus all ` + --tmpfs "/tmp:rw,noexec,nosuid,size=2g" ` + --health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" ` + --health-interval 5s ` + --health-timeout 3s ` + --health-start-period 30s ` + --health-retries 30 ` + -v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") ` + $baseImage ` + tritonserver ` + --model-repository=/models ` + --model-control-mode=explicit ` + --load-model=rf_detr_large ` + --load-model=rf_detr_large_native_kb4 ` + --disable-auto-complete-config ` + --strict-readiness=true ` + --exit-on-error=true ` + --allow-http=true ` + --allow-grpc=false ` + --allow-metrics=false *> $null + Assert-LastExitCode "M48N comparison Triton creation" + & docker start $tritonName *> $null + Assert-LastExitCode "M48N comparison Triton start" + $ready = $false + foreach ($attempt in 1..90) { + Start-Sleep -Seconds 2 + $comparisonContainer = Get-Container $tritonName + if (-not $comparisonContainer.State.Running) { + & docker logs $tritonName + throw "M48N comparison Triton stopped during startup" + } + if ($comparisonContainer.State.Health.Status -ceq "healthy") { + $ready = $true + break + } + } + if (-not $ready) { throw "M48N comparison Triton did not become healthy" } + + $runnerArguments = @( + "run", + "--name", $runnerName, + "--label", "com.nodedc.product=mission-core", + "--label", "com.nodedc.stack=ndc-mission-core-compute", + "--label", "com.nodedc.role=bounded-rf-detr-native-vs-704-runner", + "--label", "com.nodedc.managed-by=codex-bounded-experiment", + "--network", ("container:{0}" -f $tritonName), + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "512", + "--shm-size", "1g", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", "PYTHONPATH=/opt/opencv:/opt/parity", + "-v", ($runtimeVolume + ":/opt/parity:ro"), + "-v", ((Convert-ToDockerPath $opencvPackages) + ":/opt/opencv:ro"), + "-v", ((Convert-ToDockerPath $release) + ":/release:ro"), + "-v", ((Convert-ToDockerPath $video) + ":/input/video.mp4:ro"), + "-v", ((Convert-ToDockerPath $mask) + ":/input/mask.png:ro"), + "-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"), + "--entrypoint", "python3", + $baseImage, + "/release/run_m48n_native_vs_704_worker.py", + "--video", "/input/video.mp4", + "--expected-video-sha256", $videoSha256, + "--mask", "/input/mask.png", + "--expected-mask-sha256", $maskSha256, + "--baseline-triton-origin", "http://127.0.0.1:8000", + "--native-triton-origin", "http://127.0.0.1:8000", + "--baseline-engine-sha256", $baselineEngineSha256, + "--native-engine-sha256", $nativeEngineSha256, + "--output", "/output/result.json", + "--frames", "/output/frames.jsonl" + ) + if ($MaximumFrames -gt 0) { + $runnerArguments += @("--maximum-frames", [string]$MaximumFrames) + } + & docker @runnerArguments + Assert-LastExitCode "M48N native/704 RAVNOVES00 comparison" + if (-not (Test-Path -LiteralPath (Join-Path $runOutput "result.json") -PathType Leaf)) { + throw "M48N native/704 comparison result was not written" + } +} finally { + foreach ($name in @($runnerName, $tritonName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + & docker rm -f $name *> $null + } + } + $historicalAfter = Get-Container "ndc-mission-core-triton" + if ( + $historicalAfter.Id -cne $historicalId -or + -not $historicalAfter.State.Running -or + $historicalAfter.State.Health.Status -cne "healthy" + ) { + throw "Canonical Triton changed during M48N native/704 comparison" + } +} + +Write-Output ("M48N_NATIVE_VS_704_RESULT={0}" -f (Join-Path $runOutput "result.json")) +Write-Output "CANONICAL_TRITON_ACTION=none" +Write-Output "SEMANTIC_AUTHORITY_CHANGED=false" diff --git a/experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt b/experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt new file mode 100644 index 0000000..e6c7b4b --- /dev/null +++ b/experiments/perception/worker/rf_detr_large_native_kb4_config.pbtxt @@ -0,0 +1,47 @@ +name: "rf_detr_large_native_kb4" +platform: "tensorrt_plan" +max_batch_size: 0 + +input [ + { + name: "raw_kb4_bgr" + data_type: TYPE_UINT8 + dims: [ 1, 600, 800, 3 ] + } +] + +output [ + { + name: "dets" + data_type: TYPE_FP16 + dims: [ 1, 300, 4 ] + }, + { + name: "labels" + data_type: TYPE_FP16 + dims: [ 1, 300, 91 ] + } +] + +instance_group [ + { + count: 1 + kind: KIND_GPU + gpus: [ 0 ] + } +] + +model_warmup [ + { + name: "rf_detr_large_native_kb4_zero" + batch_size: 0 + inputs: { + key: "raw_kb4_bgr" + value: { + data_type: TYPE_UINT8 + dims: [ 1, 600, 800, 3 ] + zero_data: true + } + } + } +] diff --git a/experiments/perception/wrap_m48n_rf_detr_native_uint8_onnx.py b/experiments/perception/wrap_m48n_rf_detr_native_uint8_onnx.py new file mode 100644 index 0000000..ff6dbcf --- /dev/null +++ b/experiments/perception/wrap_m48n_rf_detr_native_uint8_onnx.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Fuse raw KB4 UINT8 preprocessing into the native RF-DETR ONNX graph.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Final + +import numpy as np +import onnx # type: ignore[import-not-found] +from onnx import TensorProto, helper, numpy_helper +from PIL import Image + +SCHEMA_VERSION: Final = "missioncore.m48n-rf-detr-native-uint8-wrapper/v0" +PROFILE_ID: Final = "rf-detr-large-coco-native-kb4-uint8-trt11-fp16/v0" +RAW_INPUT_NAME: Final = "raw_kb4_bgr" +SOURCE_HEIGHT: Final = 600 +SOURCE_WIDTH: Final = 800 +MODEL_HEIGHT: Final = 608 +MODEL_WIDTH: Final = 800 +FILL_VALUE: Final = 114 +MEANS: Final = (0.485, 0.456, 0.406) +STDS: Final = (0.229, 0.224, 0.225) +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-input-sha256", required=True) + parser.add_argument("--mask", type=Path, required=True) + parser.add_argument("--expected-mask-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + arguments = parser.parse_args() + + source = arguments.input.resolve(strict=True) + source_sha256 = sha256_path(source) + if source_sha256 != arguments.expected_input_sha256: + raise RuntimeError("FP16 core SHA-256 does not match the conversion manifest") + mask_path = arguments.mask.resolve(strict=True) + mask_sha256 = sha256_path(mask_path) + if mask_sha256 != arguments.expected_mask_sha256: + raise RuntimeError("valid-FOV mask SHA-256 changed") + output = arguments.output.absolute() + manifest = arguments.manifest.absolute() + if output.exists() or manifest.exists(): + raise RuntimeError("wrapped ONNX output or manifest already exists") + + started_utc_ns = time.time_ns() + mask = np.asarray(Image.open(mask_path).convert("L")) > 0 + if mask.shape != (SOURCE_HEIGHT, SOURCE_WIDTH) or not np.any(mask): + raise RuntimeError("valid-FOV mask geometry changed") + graph = onnx.load(str(source)) + _attach_uint8_preprocessing(graph, np.asarray(mask, dtype=np.bool_)) + onnx.checker.check_model(graph) + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + onnx.save(graph, str(output)) + verified = onnx.load(str(output), load_external_data=False) + onnx.checker.check_model(verified) + inputs = [_tensor_description(value) for value in verified.graph.input] + outputs = [_tensor_description(value) for value in verified.graph.output] + expected_input = [ + { + "name": RAW_INPUT_NAME, + "element_type": TensorProto.UINT8, + "shape": [1, SOURCE_HEIGHT, SOURCE_WIDTH, 3], + } + ] + if inputs != expected_input: + raise RuntimeError(f"wrapped native RF-DETR input boundary changed: {inputs}") + if {item["name"]: item["element_type"] for item in outputs} != { + "dets": TensorProto.FLOAT16, + "labels": TensorProto.FLOAT16, + }: + raise RuntimeError(f"wrapped native RF-DETR outputs changed: {outputs}") + operations = [node.op_type for node in verified.graph.node[:7]] + if operations != ["Cast", "Mul", "Add", "Pad", "Gather", "Transpose", "Mul"]: + raise RuntimeError(f"native preprocessing graph prefix changed: {operations}") + if verified.graph.node[7].op_type != "Add": + raise RuntimeError("native normalization bias node is missing") + + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": PROFILE_ID, + "source_fp16_onnx_sha256": source_sha256, + "valid_fov_mask_sha256": mask_sha256, + "output_onnx_sha256": sha256_path(output), + "output_size_bytes": output.stat().st_size, + "inputs": inputs, + "outputs": outputs, + "preprocessing": { + "execution_device": "TensorRT GPU graph", + "source_raster_wh": [SOURCE_WIDTH, SOURCE_HEIGHT], + "source_color": "BGR", + "model_color": "RGB", + "valid_fov_fill_value": FILL_VALUE, + "padding_tblr": [0, 8, 0, 0], + "model_canvas_wh": [MODEL_WIDTH, MODEL_HEIGHT], + "normalization_mean": list(MEANS), + "normalization_std": list(STDS), + "resize": False, + "rectification": False, + "warp": False, + "crop": False, + }, + "transport": { + "datatype": "UINT8", + "layout": "NHWC", + "bytes_per_frame": SOURCE_HEIGHT * SOURCE_WIDTH * 3, + }, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": FALSE_AUTHORITY, + } + manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest.write_bytes(canonical_json(document) + b"\n") + print(output) + print(json.dumps(document, indent=2, sort_keys=True)) + return 0 + + +def _attach_uint8_preprocessing(graph: Any, mask: np.ndarray[Any, np.dtype[np.bool_]]) -> None: + old_input = next((item for item in graph.graph.input if item.name == "input"), None) + if old_input is None: + raise RuntimeError("FP16 core has no input tensor named 'input'") + if _tensor_description(old_input) != { + "name": "input", + "element_type": TensorProto.FLOAT, + "shape": [1, 3, MODEL_HEIGHT, MODEL_WIDTH], + }: + raise RuntimeError("FP16 core input contract changed") + boundary_cast = next( + ( + node + for node in graph.graph.node + if node.name == "missioncore_input_fp32_to_fp16" + ), + None, + ) + if ( + boundary_cast is None + or boundary_cast.op_type != "Cast" + or list(boundary_cast.input) != ["input"] + or list(boundary_cast.output) != ["missioncore_input_fp16"] + ): + raise RuntimeError("FP16 core boundary cast changed") + + graph.graph.node.remove(boundary_cast) + graph.graph.input.remove(old_input) + graph.graph.input.insert( + 0, + helper.make_tensor_value_info( + RAW_INPUT_NAME, + TensorProto.UINT8, + [1, SOURCE_HEIGHT, SOURCE_WIDTH, 3], + ), + ) + + valid = mask.astype(np.float16)[None, :, :, None] + invalid_fill = ((~mask).astype(np.float16) * FILL_VALUE)[None, :, :, None] + scale = np.asarray( + [1.0 / (255.0 * value) for value in STDS], + dtype=np.float16, + ).reshape(1, 3, 1, 1) + bias = np.asarray( + [-mean / std for mean, std in zip(MEANS, STDS, strict=True)], + dtype=np.float16, + ).reshape(1, 3, 1, 1) + initializers = ( + numpy_helper.from_array(valid, name="missioncore_valid_fov_fp16"), + numpy_helper.from_array(invalid_fill, name="missioncore_invalid_fill_fp16"), + numpy_helper.from_array( + np.asarray((0, 0, 0, 0, 0, 8, 0, 0), dtype=np.int64), + name="missioncore_bottom_pad", + ), + numpy_helper.from_array( + np.asarray(FILL_VALUE, dtype=np.float16), + name="missioncore_pad_fill_fp16", + ), + numpy_helper.from_array( + np.asarray((2, 1, 0), dtype=np.int64), + name="missioncore_bgr_to_rgb_indices", + ), + numpy_helper.from_array(scale, name="missioncore_imagenet_scale_fp16"), + numpy_helper.from_array(bias, name="missioncore_imagenet_bias_fp16"), + ) + graph.graph.initializer.extend(initializers) + + nodes = ( + helper.make_node( + "Cast", + inputs=[RAW_INPUT_NAME], + outputs=["missioncore_raw_fp16"], + name="missioncore_raw_uint8_to_fp16", + to=TensorProto.FLOAT16, + ), + helper.make_node( + "Mul", + inputs=["missioncore_raw_fp16", "missioncore_valid_fov_fp16"], + outputs=["missioncore_valid_pixels_fp16"], + name="missioncore_apply_valid_fov", + ), + helper.make_node( + "Add", + inputs=["missioncore_valid_pixels_fp16", "missioncore_invalid_fill_fp16"], + outputs=["missioncore_masked_bgr_fp16"], + name="missioncore_fill_invalid_fov", + ), + helper.make_node( + "Pad", + inputs=[ + "missioncore_masked_bgr_fp16", + "missioncore_bottom_pad", + "missioncore_pad_fill_fp16", + ], + outputs=["missioncore_padded_bgr_fp16"], + name="missioncore_pad_bottom_to_608", + mode="constant", + ), + helper.make_node( + "Gather", + inputs=["missioncore_padded_bgr_fp16", "missioncore_bgr_to_rgb_indices"], + outputs=["missioncore_padded_rgb_fp16"], + name="missioncore_bgr_to_rgb", + axis=3, + ), + helper.make_node( + "Transpose", + inputs=["missioncore_padded_rgb_fp16"], + outputs=["missioncore_nchw_rgb_fp16"], + name="missioncore_nhwc_to_nchw", + perm=[0, 3, 1, 2], + ), + helper.make_node( + "Mul", + inputs=["missioncore_nchw_rgb_fp16", "missioncore_imagenet_scale_fp16"], + outputs=["missioncore_scaled_rgb_fp16"], + name="missioncore_imagenet_scale", + ), + helper.make_node( + "Add", + inputs=["missioncore_scaled_rgb_fp16", "missioncore_imagenet_bias_fp16"], + outputs=["missioncore_input_fp16"], + name="missioncore_imagenet_bias", + ), + ) + for node in reversed(nodes): + graph.graph.node.insert(0, node) + + +def _tensor_description(value: Any) -> dict[str, object]: + tensor = value.type.tensor_type + return { + "name": value.name, + "element_type": tensor.elem_type, + "shape": [_dimension_value(item) for item in tensor.shape.dim], + } + + +def _dimension_value(value: Any) -> int | str | None: + if value.HasField("dim_value"): + return int(value.dim_value) + if value.HasField("dim_param"): + return str(value.dim_param) + return None + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main())