From 33cef2fdea13bfcc805de224a563db9dde2c7f7a Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Tue, 25 Aug 2026 16:44:15 +0300 Subject: [PATCH] feat(perception): evaluate fixed-class detector candidates --- .../fixed-class-detector-tournament-v0.json | 89 +++ .../rf-detr-large-risk-shadow-v0.json | 81 +++ .../yolox-s-all-coco-shadow-v2.json | 60 ++ ..._FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md | 133 +++++ .../perception/M48S_YOLOX_ALL_COCO_REPORT.md | 62 ++ .../convert_m48s_rf_detr_onnx_fp16.py | 156 +++++ .../perception/export_m48s_rf_detr_worker.py | 147 +++++ .../run_m48s_all_coco_yolox_worker.py | 288 +++++++++ ...un_m48s_fixed_detector_candidate_worker.py | 546 ++++++++++++++++++ .../run_m48s_rf_detr_load_worker.py | 417 +++++++++++++ .../run_m48s_rf_detr_triton_worker.py | 508 ++++++++++++++++ .../seal_m48s_fixed_detector_tournament.py | 242 ++++++++ .../seal_m48s_rf_detr_deployment.py | 264 +++++++++ .../Dockerfile.fixed-detector-tournament | 30 + .../bootstrap_fixed_detector_tournament.sh | 28 + .../worker/build_m48s_rf_detr_tensorrt.sh | 30 + .../worker/rf_detr_large_config.pbtxt | 47 ++ src/k1link/perception/detector.py | 158 ++++- .../fixed_class_detector_tournament.py | 285 +++++++++ src/k1link/perception/recorded_source.py | 18 +- .../perception/rf_detr_object_detector.py | 401 +++++++++++++ .../perception/yolox_object_detector.py | 60 +- tests/test_fixed_class_detector_tournament.py | 80 +++ tests/test_m48s_rf_detr_deployment_result.py | 60 ++ tests/test_m48s_yolox_all_coco_result.py | 43 ++ tests/test_rf_detr_detector_provider.py | 202 +++++++ tests/test_yolox_all_coco_profile.py | 29 + tests/test_yolox_detector_provider.py | 47 ++ 28 files changed, 4498 insertions(+), 13 deletions(-) create mode 100644 config/perception/fixed-class-detector-tournament-v0.json create mode 100644 config/perception/rf-detr-large-risk-shadow-v0.json create mode 100644 config/perception/yolox-s-all-coco-shadow-v2.json create mode 100644 experiments/perception/M48S_FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md create mode 100644 experiments/perception/M48S_YOLOX_ALL_COCO_REPORT.md create mode 100644 experiments/perception/convert_m48s_rf_detr_onnx_fp16.py create mode 100644 experiments/perception/export_m48s_rf_detr_worker.py create mode 100644 experiments/perception/run_m48s_all_coco_yolox_worker.py create mode 100644 experiments/perception/run_m48s_fixed_detector_candidate_worker.py create mode 100644 experiments/perception/run_m48s_rf_detr_load_worker.py create mode 100644 experiments/perception/run_m48s_rf_detr_triton_worker.py create mode 100644 experiments/perception/seal_m48s_fixed_detector_tournament.py create mode 100644 experiments/perception/seal_m48s_rf_detr_deployment.py create mode 100644 experiments/perception/worker/Dockerfile.fixed-detector-tournament create mode 100644 experiments/perception/worker/bootstrap_fixed_detector_tournament.sh create mode 100644 experiments/perception/worker/build_m48s_rf_detr_tensorrt.sh create mode 100644 experiments/perception/worker/rf_detr_large_config.pbtxt create mode 100644 src/k1link/perception/fixed_class_detector_tournament.py create mode 100644 src/k1link/perception/rf_detr_object_detector.py create mode 100644 tests/test_fixed_class_detector_tournament.py create mode 100644 tests/test_m48s_rf_detr_deployment_result.py create mode 100644 tests/test_m48s_yolox_all_coco_result.py create mode 100644 tests/test_rf_detr_detector_provider.py create mode 100644 tests/test_yolox_all_coco_profile.py diff --git a/config/perception/fixed-class-detector-tournament-v0.json b/config/perception/fixed-class-detector-tournament-v0.json new file mode 100644 index 0000000..168525a --- /dev/null +++ b/config/perception/fixed-class-detector-tournament-v0.json @@ -0,0 +1,89 @@ +{ + "schema_version": "missioncore.fixed-class-detector-tournament-profile/v0", + "profile_id": "m48s-fixed-class-detector-tournament/v0", + "source": { + "slice_id": "m48s-risk-11-valid-fov-fill/v1", + "frame_count": 11, + "valid_fov_result_id": "valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2", + "baseline_result_id": "m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06" + }, + "candidates": [ + { + "profile_id": "dfine-s-coco-640-fp16/v0", + "provider_id": "shadow-dfine-s-coco/v0", + "role": "fast", + "upstream": { + "repository": "https://github.com/Peterande/D-FINE", + "revision": "956d1709314c2c6a4df6f34de232054578a7449f", + "license": "Apache-2.0" + }, + "checkpoint": { + "url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_coco.pth", + "training_basis": "COCO-only", + "sha256": "48a6c8cc43eb57186843f752e2e8461ddd3326e0d3c575e71e6e960844683e89" + }, + "model": { + "dataset_vocabulary": "COCO-80", + "input_width": 640, + "input_height": 640, + "published_coco_ap_50_95": 48.5, + "published_t4_tensorrt_fp16_ms": 3.49, + "published_parameter_count_millions": 10 + }, + "deployment_path": ["PyTorch qualification", "ONNX", "TensorRT FP16", "Triton"] + }, + { + "profile_id": "rf-detr-large-coco-704-fp16/v0", + "provider_id": "shadow-rf-detr-large-coco/v0", + "role": "strong", + "upstream": { + "repository": "https://github.com/roboflow/rf-detr", + "version": "1.9.4", + "revision": "9b009fa928d6218320439803d1da01869a85c072", + "license": "Apache-2.0" + }, + "checkpoint": { + "url": "https://storage.googleapis.com/rfdetr/rf-detr-large-2026.pth", + "training_basis": "Apache-designated COCO checkpoint", + "expected_md5": "5cb72153541cbcb9aa6efa26222acc75", + "sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38" + }, + "model": { + "dataset_vocabulary": "COCO-80", + "input_width": 704, + "input_height": 704, + "published_coco_ap_50_95": 56.5, + "published_t4_tensorrt_fp16_ms": 6.8, + "published_parameter_count_millions": 33.9 + }, + "deployment_path": ["PyTorch qualification", "ONNX", "TensorRT FP16", "Triton"] + } + ], + "emission": { + "class_vocabulary": "COCO-80", + "qualification_score_thresholds": [0.25, 0.5], + "geometry_owns_occupancy": true, + "static_object_names_have_behavior_authority": false, + "inference_passes_per_frame": 1 + }, + "risk_policy": { + "classified_groups": ["person", "animal", "light-road-user", "vehicle"], + "unknown_moving_response": "conservative-risk", + "unknown_stationary_response": "route-around" + }, + "promotion_gates": { + "minimum_recorded_source_fps": 9.5, + "maximum_world_state_p95_ms": 175, + "maximum_worker_vram_gib": 20, + "sustained_gpu_utilization_below_percent": 100, + "queue_policy": "bounded-latest-wins", + "full_load_duration_minutes": [30, 60] + }, + "authority": { + "ground_truth": false, + "candidate_accepted": false, + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false + } +} diff --git a/config/perception/rf-detr-large-risk-shadow-v0.json b/config/perception/rf-detr-large-risk-shadow-v0.json new file mode 100644 index 0000000..187ee9d --- /dev/null +++ b/config/perception/rf-detr-large-risk-shadow-v0.json @@ -0,0 +1,81 @@ +{ + "schema_version": "missioncore.rf-detr-risk-shadow-profile/v0", + "profile_id": "rf-detr-large-coco-704-trt11-fp16-risk-shadow/v0", + "provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0", + "model": { + "model_id": "rf_detr_large", + "model_version": 1, + "upstream_version": "1.9.4", + "upstream_revision": "9b009fa928d6218320439803d1da01869a85c072", + "checkpoint_sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38", + "exported_onnx_sha256": "9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300", + "strongly_typed_fp16_onnx_sha256": "9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb", + "worker_006_rtx4090_tensorrt_11_engine_sha256": "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8", + "input": { + "name": "input", + "datatype": "FP32", + "shape": [1, 3, 704, 704] + }, + "outputs": [ + {"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]}, + {"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]} + ] + }, + "preprocessing": { + "source_raster": [800, 600], + "source_color": "BGR", + "model_color": "RGB", + "valid_fov_fill_value": 114, + "resize": "704x704-bilinear-antialias-false", + "normalization_mean": [0.485, 0.456, 0.406], + "normalization_std": [0.229, 0.224, 0.225] + }, + "emission": { + "single_inference_per_source_frame": true, + "minimum_score": 0.25, + "maximum_topk_query_class_pairs": 300, + "behavior_relevant_classes": [ + "person", + "bicycle", + "car", + "motorcycle", + "bus", + "truck", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "skateboard" + ], + "geometry_owns_static_occupancy": true, + "unlisted_semantic_classes_emitted": false, + "minimum_box_area_pixels": 64, + "maximum_box_area_fraction": 0.5, + "minimum_valid_fov_fraction": 0.5, + "require_center_inside_valid_fov": true + }, + "queue": { + "policy": "bounded-latest-wins", + "capacity": 2 + }, + "status": { + "tournament_finalist": true, + "tensorrt_parity_passed": true, + "detector_load_gate_passed": true, + "integrated_world_state_gate_passed": false, + "production_accepted": false + }, + "authority": { + "ground_truth": false, + "candidate_accepted": false, + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false + } +} diff --git a/config/perception/yolox-s-all-coco-shadow-v2.json b/config/perception/yolox-s-all-coco-shadow-v2.json new file mode 100644 index 0000000..85581cc --- /dev/null +++ b/config/perception/yolox-s-all-coco-shadow-v2.json @@ -0,0 +1,60 @@ +{ + "schema_version": "missioncore.yolox-detector-profile/v2", + "profile_id": "yolox-s-raw-kb4-all-coco-shadow/v2", + "provider_id": "triton-yolox-s-raw-kb4-all-coco/v2", + "model": { + "model_id": "yolox_s:1", + "model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063", + "dataset_vocabulary": "COCO-80", + "additional_inference_passes": 0 + }, + "preprocess": { + "source_width": 800, + "source_height": 600, + "input_width": 640, + "input_height": 640, + "fill_value": 114, + "valid_fov_mask_sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" + }, + "postprocess": { + "minimum_score": 0.5, + "nms_iou_threshold": 0.45, + "target_class_ids": [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + ], + "minimum_box_area_pixels": 64.0, + "maximum_box_area_fraction": 0.5, + "minimum_valid_fov_fraction": 0.5, + "require_center_inside_valid_fov": true + }, + "class_policy": { + "emission": "all-qualified-coco-classes", + "behavior_authority": "separate-risk-policy-only", + "risk_groups": { + "person": ["person"], + "animal": [ + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", + "zebra", "giraffe" + ], + "light-road-user": ["bicycle", "motorcycle", "skateboard"], + "vehicle": ["car", "bus", "truck"] + }, + "unmapped_label_response": "advisory-only", + "unknown_moving_response": "conservative-risk", + "unknown_stationary_response": "route-around" + }, + "authority": { + "ground_truth": false, + "candidate_accepted": false, + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false + } +} diff --git a/experiments/perception/M48S_FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md b/experiments/perception/M48S_FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md new file mode 100644 index 0000000..dfc3915 --- /dev/null +++ b/experiments/perception/M48S_FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md @@ -0,0 +1,133 @@ +# M48S fixed-class detector tournament и Worker deployment gate + +Дата: 2026-08-25 +Режим: experimental shadow +Worker: `worker-006`, NVIDIA GeForce RTX 4090 +Production acceptance: **нет** + +## Решение + +RF-DETR-L выиграл bounded-турнир, был экспортирован в ONNX, преобразован в +strongly typed FP16 graph, собран TensorRT 11 и проверен через изолированный +Triton. Detector-only нагрузочный gate длиной 30 минут принят. Кандидат готов к +следующему shadow-gate внутри полного reference graph, но не получил +navigation, safety, command или actuation authority. + +Production critical path не включает Mask Grounding DINO, SAM 2 или OpenCLIP. +Статические неизвестные объекты остаются ответственностью geometry/occupancy и +объезжаются без расхода detector inference на их название. Один fixed-class +проход камеры используется только для поведенчески значимых классов. + +## Поведенческая граница + +RF-DETR shadow provider выпускает только: + +- `person`; +- `bicycle`, `motorcycle`, `skateboard`; +- `car`, `bus`, `truck`; +- COCO animal classes, включая `dog`. + +Урны, столбы, полусферы, бордюры и прочие статические препятствия не обязаны +получать семантическое имя: их наличие и геометрия принадлежат class-free +occupancy. Неизвестный движущийся объект остаётся conservative. Отдельного +COCO-класса `scooter` нет, поэтому самокат пока нельзя считать надёжно +классифицированным: до отдельного admission gate он остаётся geometry/motion +hazard, а не безопасным отрицанием. + +## Турнир на immutable 11-frame slice + +Все профили выполняли один inference pass на кадр; ручная проверка не объявлена +ground truth. + +| Профиль | Core capacity | p95 | Собака на frame 253 | Решение | +|---|---:|---:|---|---| +| YOLOX-S all-COCO/v2 | 39,136 FPS | 38,267 мс | нет | regression baseline | +| D-FINE-S COCO FP16 | 31,100 FPS | 42,945 мс | нет при 0,25 и 0,5 | отклонён | +| RF-DETR-L COCO FP16 | 44,786 FPS | 32,588 мс | да, score 0,740723 | finalist | + +D-FINE также давал заметные semantic confusions: собака как `skateboard`, корпус +сканера как `surfboard`, дублирующиеся risk-labels на одном объекте. RF-DETR на +этом slice дал более чистые person/vehicle labels и корректную собаку. + +Immutable tournament result: +`m48s-fixed-detector-tournament-0e61d75e6dc575d53e4bb98772a41d240fe627ad642de5178beb1154636e1299`. + +## TensorRT/Triton квалификация + +Закреплены следующие identities: + +- upstream RF-DETR revision: `9b009fa928d6218320439803d1da01869a85c072`; +- checkpoint SHA-256: `0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38`; +- exported ONNX SHA-256: `9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300`; +- strongly typed FP16 ONNX SHA-256: `9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb`; +- Worker 006 TensorRT engine SHA-256: `986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8`. + +TensorRT parity на frame 253: + +- PyTorch dog score: 0,740723; +- TensorRT dog score: 0,741674; +- абсолютная разница score: 0,000951; +- box IoU: 0,990117; +- class counts при threshold 0,5 совпадают точно: 57 `car`, 8 `truck`, + 6 `person`, 1 `dog`, 1 `fire hydrant`. + +100-iteration Triton benchmark: 42,496 FPS end-to-end; mean 23,531 мс; p95 +34,156 мс. Production Triton во время проверки не изменялся: использовался +отдельный безпортовый Triton-контейнер в namespace эксперимента. + +## 30-минутный source-paced gate + +Источник: RAVNOVES00, SHA-256 +`cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8`, +10,0039 FPS. Durable background services на Worker оставались включёнными. + +| Метрика | Результат | Gate | +|---|---:|---:| +| Время | 1800,020 с | ≥ 1800 с | +| Кадры | 18 008 produced / 18 008 consumed | без потерь | +| Effective FPS | 10,004 | ≥ 9,5 | +| Detector end-to-end p95 | 32,415 мс | наблюдение | +| Detector completion age p95 | 40,621 мс | ≤ 175 мс | +| Queue | max depth 1/2, replacements 0 | bounded latest-wins | +| GPU utilization | mean 51,408%, p95 55%, max 65% | без sustained 100% | +| Worker VRAM | mean 9542,7 MiB, max 9556 MiB | ≤ 20 GiB | +| Ошибки | 0 | 0 | + +Все восемь автоматических load checks приняты. Это detector-only gate, поэтому +он не доказывает p95 полного world state, качество tracker association или +корректность risk-policy. + +Immutable deployment result: +`m48s-rf-detr-deployment-gate-2feb9e1b12a5588951ad35d63bf23cf6bdd579d54b5329d46d7696f88c444547`. + +## Реализация + +- `src/k1link/perception/rf_detr_object_detector.py` — pinned preprocessing, + Triton V2 binary HTTP backend, FP16 output validation, fixed risk-class + qualification и fail-closed FOV/area gates. +- `src/k1link/perception/detector.py` — `RfDetrShadowDetectorProvider`, один + inference pass и semantic hints без authority. +- `config/perception/rf-detr-large-risk-shadow-v0.json` — неизменяемый профиль, + threshold 0,25, bounded latest-wins queue capacity 2, geometry-owned static + occupancy. +- `experiments/perception/worker/` — воспроизводимые export/build/Triton + declarations. +- `experiments/perception/run_m48s_rf_detr_load_worker.py` — source-paced + concurrent-load gate с GPU, queue и Triton accounting. +- `experiments/perception/seal_m48s_rf_detr_deployment.py` — content-addressed + immutable seal с false authority. + +## Следующий gate + +Подключить этот provider в полный reference graph вместе с существующими +geometry observations, tracker и advisory risk-policy. На том же записанном +источнике и при сохранённых Worker services требуется: + +1. world-state p95 не более 175 мс; +2. не менее 9,5 source FPS, bounded latest-wins без неучтённых потерь; +3. стабильные track identities и conservative unknown-moving handling; +4. раздельные реакции на person/animal/light-road-user/vehicle; +5. отсутствие navigation/safety/command authority до отдельного acceptance. + +Только после этого можно решать вопрос о замене текущего production detector. +Текущий результат разрешает reference-graph shadow, а не production switch. diff --git a/experiments/perception/M48S_YOLOX_ALL_COCO_REPORT.md b/experiments/perception/M48S_YOLOX_ALL_COCO_REPORT.md new file mode 100644 index 0000000..bc46f49 --- /dev/null +++ b/experiments/perception/M48S_YOLOX_ALL_COCO_REPORT.md @@ -0,0 +1,62 @@ +# M48S YOLOX-S all-COCO shadow report + +Date: 2026-08-25 +Status: executable shadow completed; full-load promotion gate open + +## Why six classes were previously emitted + +The accepted `triton-yolox-s-raw-kb4/v1` provider was deliberately frozen on +COCO ids `0, 1, 2, 3, 5, 7`: person, bicycle, car, motorcycle, bus and truck. +That was a bounded detector qualification and reproducibility boundary, not an +inference optimization. YOLOX-S already returns an `[1, 8400, 85]` tensor with +all 80 COCO class scores. The six-class filter ran after the single inference. + +The old provider, hashes and M4 replay results remain unchanged. The new +`triton-yolox-s-raw-kb4-all-coco/v2` provider uses the same model, tensor, +preprocess, thresholds and valid-FOV gates, but emits every qualified COCO class. + +## Worker comparison + +Both profiles were applied to the exact same tensor response for each of the 11 +M48S frames. The client shared the Worker's `mission-core-compute_default` +network with Triton, avoiding host-NAT tensor transport. + +| Measure | Frozen six classes | All COCO-80 | +|---|---:|---:| +| Inference passes per frame | 1 | 1, shared | +| Detections | 44 | 45 | +| Postprocess mean, 220 balanced iterations | 6.430 ms | 6.725 ms | +| Postprocess p50 | 5.671 ms | 5.679 ms | +| Postprocess p95 | 11.013 ms | 11.592 ms | + +The measured mean postprocess difference was `0.295 ms`; the p50 difference was +`0.008 ms`. The combined preprocess + inference + all-COCO postprocess capacity +was `39.136 FPS` on this bounded slice. This is a capacity diagnostic, not a +full-route load acceptance. + +The only newly emitted detection was `handbag` on frame 253, correctly covering +the bag carried by the visible person. The class counts were 36 car, five truck, +three person and one handbag. The visible dog on frame 253 was not detected. +Removing the filter therefore exposes all model answers at negligible compute +cost, but does not repair classes the model fails to recognize. + +Immutable result: +`m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06`. + +## Policy boundary + +All qualified COCO labels are now available to downstream consumers. Emission +does not grant every class behavioral authority: + +- person, animal, light road user and vehicle labels may enter a separately + versioned risk policy after qualification; +- other labels remain advisory diagnostics; +- geometry owns occupancy for every object; +- unknown moving objects retain conservative risk; +- unknown stationary objects remain route-around; +- commands, actuation, navigation and safety authority remain false. + +The next gate is the full recorded source under representative concurrent Worker +load. It must compare source delivery, detector FPS, p95 latency, queue depth, +drops, GPU utilization and VRAM against the frozen six-class baseline before v2 +can replace v1 in the production assembly. diff --git a/experiments/perception/convert_m48s_rf_detr_onnx_fp16.py b/experiments/perception/convert_m48s_rf_detr_onnx_fp16.py new file mode 100644 index 0000000..994bee3 --- /dev/null +++ b/experiments/perception/convert_m48s_rf_detr_onnx_fp16.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Convert the pinned RF-DETR ONNX graph to a strongly typed FP16 graph.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path +from typing import Any, Final + +import onnx # type: ignore[import-not-found] +from onnx import TensorProto +from onnxconverter_common import float16 # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-onnx-fp16-conversion/v3" +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--expected-input-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + arguments = parser.parse_args() + + source = arguments.input.resolve(strict=True) + source_sha256 = sha256_path(source) + if source_sha256 != arguments.expected_input_sha256: + raise RuntimeError("source ONNX SHA-256 does not match the export manifest") + output = arguments.output.absolute() + manifest = arguments.manifest.absolute() + if output.exists() or manifest.exists(): + raise RuntimeError("FP16 ONNX output or manifest already exists") + + started_utc_ns = time.time_ns() + graph = onnx.load(str(source)) + converted = float16.convert_float_to_float16( + graph, + keep_io_types=False, + disable_shape_infer=False, + ) + retargeted_casts = _retarget_float_casts_to_fp16(converted) + _insert_fp32_input_cast(converted) + onnx.checker.check_model(converted) + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + onnx.save(converted, str(output)) + verified = onnx.load(str(output), load_external_data=False) + onnx.checker.check_model(verified) + input_types = {value.name: value.type.tensor_type.elem_type for value in verified.graph.input} + output_types = { + value.name: value.type.tensor_type.elem_type for value in verified.graph.output + } + if input_types != {"input": TensorProto.FLOAT}: + raise RuntimeError(f"FP16 ONNX input boundary is not FLOAT: {input_types}") + if output_types != {"dets": TensorProto.FLOAT16, "labels": TensorProto.FLOAT16}: + raise RuntimeError(f"FP16 ONNX outputs are not FLOAT16: {output_types}") + initializer_counts = _initializer_type_counts(verified) + if initializer_counts.get("FLOAT16", 0) == 0: + raise RuntimeError("FP16 ONNX has no FLOAT16 initializers") + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": "rf-detr-large-coco-704-trt11-fp16/v0", + "source_onnx_sha256": source_sha256, + "output_onnx_sha256": sha256_path(output), + "output_size_bytes": output.stat().st_size, + "boundary_types": {"inputs": input_types, "outputs": output_types}, + "initializer_type_counts": initializer_counts, + "float_casts_retargeted_to_fp16": retargeted_casts, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": FALSE_AUTHORITY, + } + manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest.write_bytes(canonical_json(document) + b"\n") + print(output) + print(json.dumps(document, indent=2, sort_keys=True)) + return 0 + + +def _initializer_type_counts(graph: Any) -> dict[str, int]: + counts: dict[str, int] = {} + for initializer in graph.graph.initializer: + name = TensorProto.DataType.Name(initializer.data_type) + counts[name] = counts.get(name, 0) + 1 + return dict(sorted(counts.items())) + + +def _insert_fp32_input_cast(graph: Any) -> None: + """Keep a conventional FP32 client boundary before the strongly typed FP16 graph.""" + + input_value = next((item for item in graph.graph.input if item.name == "input"), None) + if input_value is None: + raise RuntimeError("RF-DETR graph has no input tensor named 'input'") + if input_value.type.tensor_type.elem_type != TensorProto.FLOAT16: + raise RuntimeError("RF-DETR converted input is not FLOAT16 before boundary adaptation") + cast_output = "missioncore_input_fp16" + for node in graph.graph.node: + for index, name in enumerate(node.input): + if name == "input": + node.input[index] = cast_output + cast = onnx.helper.make_node( + "Cast", + inputs=["input"], + outputs=[cast_output], + name="missioncore_input_fp32_to_fp16", + to=TensorProto.FLOAT16, + ) + graph.graph.node.insert(0, cast) + input_value.type.tensor_type.elem_type = TensorProto.FLOAT + + +def _retarget_float_casts_to_fp16(graph: Any) -> int: + """Retarget explicit PyTorch FLOAT casts that would re-expand an FP16 data path.""" + + count = 0 + for node in graph.graph.node: + if node.op_type != "Cast": + continue + for attribute in node.attribute: + if attribute.name == "to" and attribute.i == TensorProto.FLOAT: + attribute.i = TensorProto.FLOAT16 + count += 1 + if count == 0: + raise RuntimeError("RF-DETR graph has no FLOAT casts to retarget") + return count + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/export_m48s_rf_detr_worker.py b/experiments/perception/export_m48s_rf_detr_worker.py new file mode 100644 index 0000000..fecba15 --- /dev/null +++ b/experiments/perception/export_m48s_rf_detr_worker.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Export the pinned M48S RF-DETR finalist to a static ONNX artifact.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from collections.abc import Iterable +from pathlib import Path +from typing import Any, Final + +import onnx # type: ignore[import-not-found] + +SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-onnx-export/v0" +FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--upstream-revision", required=True) + arguments = parser.parse_args() + + checkpoint = arguments.checkpoint.resolve(strict=True) + checkpoint_sha256 = sha256_path(checkpoint) + if checkpoint_sha256 != arguments.expected_checkpoint_sha256: + raise RuntimeError("checkpoint SHA-256 does not match the pinned finalist") + output_root = arguments.output_root.absolute() + manifest_path = arguments.manifest.absolute() + if output_root.exists(): + raise RuntimeError("ONNX output root already exists") + if manifest_path.exists(): + raise RuntimeError("ONNX export manifest already exists") + + from rfdetr import RFDETRLarge # type: ignore[import-not-found] + + started_utc_ns = time.time_ns() + model = RFDETRLarge(pretrain_weights=str(checkpoint)) + exported_path = Path( + model.export( + output_dir=str(output_root), + format="onnx", + shape=(704, 704), + batch_size=1, + dynamic_batch=False, + opset_version=17, + verbose=False, + notes={ + "missioncore_profile_id": "rf-detr-large-coco-704-fp16/v0", + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": checkpoint_sha256, + "authority": FALSE_AUTHORITY, + }, + ) + ).resolve(strict=True) + graph = onnx.load(str(exported_path), load_external_data=False) + onnx.checker.check_model(graph) + inputs = [_tensor_description(value) for value in graph.graph.input] + outputs = [_tensor_description(value) for value in graph.graph.output] + expected_input = [{"name": "input", "element_type": 1, "shape": [1, 3, 704, 704]}] + if inputs != expected_input: + raise RuntimeError(f"unexpected RF-DETR ONNX input contract: {inputs}") + if [item["name"] for item in outputs] != ["dets", "labels"]: + raise RuntimeError(f"unexpected RF-DETR ONNX outputs: {outputs}") + + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": "rf-detr-large-coco-704-fp16/v0", + "provider_id": "shadow-rf-detr-large-coco-onnx/v0", + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": checkpoint_sha256, + "onnx": { + "path": str(exported_path), + "sha256": sha256_path(exported_path), + "size_bytes": exported_path.stat().st_size, + "opset_imports": [ + {"domain": item.domain, "version": item.version} + for item in graph.opset_import + ], + "inputs": inputs, + "outputs": outputs, + }, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": FALSE_AUTHORITY, + } + manifest_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + manifest_path.write_bytes(canonical_json(document) + b"\n") + print(exported_path) + print(json.dumps(document["onnx"], indent=2, sort_keys=True)) + return 0 + + +def _tensor_description(value: Any) -> dict[str, object]: + tensor = value.type.tensor_type + return { + "name": value.name, + "element_type": tensor.elem_type, + "shape": [_dimension_value(item) for item in tensor.shape.dim], + } + + +def _dimension_value(value: Any) -> int | str | None: + if value.HasField("dim_value"): + return int(value.dim_value) + if value.HasField("dim_param"): + return str(value.dim_param) + return None + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _names(values: Iterable[dict[str, object]]) -> tuple[object, ...]: + """Keep static analyzers honest when ONNX collections are inspected in tests.""" + + return tuple(value.get("name") for value in values) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48s_all_coco_yolox_worker.py b/experiments/perception/run_m48s_all_coco_yolox_worker.py new file mode 100644 index 0000000..05b4774 --- /dev/null +++ b/experiments/perception/run_m48s_all_coco_yolox_worker.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Compare frozen road classes with all COCO classes on one YOLOX tensor pass.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from collections import Counter +from pathlib import Path +from typing import Any, Final + +import numpy as np +from PIL import Image + +from k1link.perception.yolox_object_detector import ( + ALL_COCO_YOLOX_CONFIG, + COCO_CLASSES, + FROZEN_YOLOX_CONFIG, + YOLOX_MODEL_SHA256, + TritonHttpInferenceBackend, + YoloxPostprocessConfig, + load_valid_fov_mask, + postprocess_yolox, + preprocess_raw_kb4, +) + +SCHEMA: Final = "missioncore.m48s-yolox-all-coco-shadow/v0" +POSTPROCESS_BENCHMARK_ITERATIONS: Final = 20 +AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--valid-fov-mask", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument( + "--triton-endpoint", + default="http://host.docker.internal:8000", + ) + arguments = parser.parse_args() + input_root = arguments.input_root.resolve(strict=True) + images = tuple(sorted(input_root.glob("frame-*.jpg"))) + if len(images) != 11: + raise RuntimeError("M48S all-COCO shadow requires the exact 11-frame slice") + output_parent = arguments.output_root.expanduser().absolute() + output_parent.mkdir(mode=0o700, parents=True, exist_ok=True) + mask = load_valid_fov_mask(arguments.valid_fov_mask) + backend = TritonHttpInferenceBackend(arguments.triton_endpoint) + rows = [] + totals: Counter[str] = Counter() + timings: dict[str, list[float]] = { + "decode_ms": [], + "preprocess_ms": [], + "inference_ms": [], + "frozen_postprocess_ms": [], + "all_coco_postprocess_ms": [], + "all_coco_core_ms": [], + } + postprocess_benchmark: dict[str, list[float]] = { + "frozen_ms": [], + "all_coco_ms": [], + } + try: + for image_path in images: + started = time.perf_counter_ns() + with Image.open(image_path) as opened: + rgb = np.asarray(opened.convert("RGB"), dtype=np.uint8) + bgr = np.ascontiguousarray(rgb[:, :, ::-1]) + decoded = time.perf_counter_ns() + tensor = preprocess_raw_kb4(bgr, mask, config=ALL_COCO_YOLOX_CONFIG) + preprocessed = time.perf_counter_ns() + output = backend.infer(tensor) + inferred = time.perf_counter_ns() + frozen = postprocess_yolox(output, mask, config=FROZEN_YOLOX_CONFIG) + frozen_postprocessed = time.perf_counter_ns() + all_coco = postprocess_yolox(output, mask, config=ALL_COCO_YOLOX_CONFIG) + all_postprocessed = time.perf_counter_ns() + _benchmark_postprocess( + output, + mask, + destination=postprocess_benchmark, + ) + frozen_ids = { + _detection_identity(item.class_id, item.score, item.bbox_xyxy) + for item in frozen.detections + } + added = tuple( + item + for item in all_coco.detections + if _detection_identity(item.class_id, item.score, item.bbox_xyxy) + not in frozen_ids + ) + totals["frame_count"] += 1 + totals["frozen_detection_count"] += len(frozen.detections) + totals["all_coco_detection_count"] += len(all_coco.detections) + totals["added_detection_count"] += len(added) + for item in all_coco.detections: + totals[f"class:{item.label}"] += 1 + timings["decode_ms"].append(_milliseconds(started, decoded)) + timings["preprocess_ms"].append(_milliseconds(decoded, preprocessed)) + timings["inference_ms"].append(_milliseconds(preprocessed, inferred)) + timings["frozen_postprocess_ms"].append( + _milliseconds(inferred, frozen_postprocessed) + ) + timings["all_coco_postprocess_ms"].append( + _milliseconds(frozen_postprocessed, all_postprocessed) + ) + timings["all_coco_core_ms"].append( + _milliseconds(decoded, preprocessed) + + _milliseconds(preprocessed, inferred) + + _milliseconds(frozen_postprocessed, all_postprocessed) + ) + rows.append( + { + "frame_name": image_path.name, + "source_sha256": _sha256(image_path), + "frozen_detections": [ + _detection_document(item) for item in frozen.detections + ], + "all_coco_detections": [ + _detection_document(item) for item in all_coco.detections + ], + "added_detections": [_detection_document(item) for item in added], + "timing_ms": {name: values[-1] for name, values in timings.items()}, + "authority": AUTHORITY, + } + ) + finally: + backend.close() + frame_bytes = b"".join(_canonical_json(item) + b"\n" for item in rows) + timing_metrics = {name: _timing_summary(values) for name, values in timings.items()} + mean_core_ms = timing_metrics["all_coco_core_ms"]["mean"] + metrics = { + "frames": {"requested": 11, "completed": totals["frame_count"]}, + "inference_passes_per_frame": 1, + "frozen_detection_count": totals["frozen_detection_count"], + "all_coco_detection_count": totals["all_coco_detection_count"], + "added_detection_count": totals["added_detection_count"], + "all_coco_class_counts": { + key.removeprefix("class:"): value + for key, value in sorted(totals.items()) + if key.startswith("class:") + }, + "timing_ms": timing_metrics, + "postprocess_benchmark": { + "iterations_per_profile_per_frame": POSTPROCESS_BENCHMARK_ITERATIONS, + "timing_ms": { + name: _timing_summary(values) + for name, values in postprocess_benchmark.items() + }, + }, + "all_coco_core_capacity_fps": round(1000.0 / mean_core_ms, 6), + "authority": AUTHORITY, + } + identity = { + "schema_version": SCHEMA, + "model_sha256": YOLOX_MODEL_SHA256, + "class_count": len(COCO_CLASSES), + "frozen_target_class_ids": list(FROZEN_YOLOX_CONFIG.target_class_ids), + "all_coco_target_class_ids": list(ALL_COCO_YOLOX_CONFIG.target_class_ids), + "valid_fov_mask_sha256": _sha256(arguments.valid_fov_mask), + "producer_sha256": _sha256(Path(__file__)), + "frames_sha256": hashlib.sha256(frame_bytes).hexdigest(), + "metrics": metrics, + "completed": totals["frame_count"] == 11, + "accepted": False, + "authority": AUTHORITY, + } + result_id = "m48s-yolox-all-coco-shadow-" + hashlib.sha256( + _canonical_json(identity) + ).hexdigest() + destination = output_parent / result_id + if destination.exists(): + raise RuntimeError("immutable M48S all-COCO result already exists") + destination.mkdir(mode=0o700) + (destination / "frames.jsonl").write_bytes(frame_bytes) + (destination / "manifest.json").write_bytes( + _canonical_json({"result_id": result_id, **identity}) + b"\n" + ) + (destination / "report.json").write_bytes( + _canonical_json( + { + "schema_version": SCHEMA, + "result_id": result_id, + "completed": identity["completed"], + "accepted": False, + "metrics": metrics, + "decision": { + "all_coco_emission_completed": True, + "additional_inference_passes": 0, + "navigation_or_safety_accepted": False, + "next_gate": "full-load all-COCO detector replay", + }, + "authority": AUTHORITY, + } + ) + + b"\n" + ) + print(result_id) + print(json.dumps(metrics, indent=2, sort_keys=True)) + return 0 + + +def _detection_identity( + class_id: int, + score: float, + box: tuple[float, float, float, float], +) -> tuple[int, float, tuple[float, float, float, float]]: + return class_id, score, box + + +def _benchmark_postprocess( + output: np.ndarray[Any, Any], + mask: np.ndarray[Any, Any], + *, + destination: dict[str, list[float]], +) -> None: + for iteration in range(POSTPROCESS_BENCHMARK_ITERATIONS): + profiles: tuple[tuple[str, YoloxPostprocessConfig], ...] + if iteration % 2: + profiles = ( + ("frozen_ms", FROZEN_YOLOX_CONFIG), + ("all_coco_ms", ALL_COCO_YOLOX_CONFIG), + ) + else: + profiles = ( + ("all_coco_ms", ALL_COCO_YOLOX_CONFIG), + ("frozen_ms", FROZEN_YOLOX_CONFIG), + ) + for name, profile in profiles: + started = time.perf_counter_ns() + postprocess_yolox(output, mask, config=profile) + completed = time.perf_counter_ns() + destination[name].append(_milliseconds(started, completed)) + + +def _detection_document(item: Any) -> dict[str, object]: + return { + "class_id": item.class_id, + "label": item.label, + "score": item.score, + "bbox_xyxy": list(item.bbox_xyxy), + "valid_fov_fraction": item.valid_fov_fraction, + } + + +def _milliseconds(started: int, completed: int) -> float: + return round(max(0, completed - started) / 1_000_000.0, 6) + + +def _timing_summary(values: list[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "mean": round(float(array.mean()), 6), + "p50": round(float(np.percentile(array, 50)), 6), + "p95": round(float(np.percentile(array, 95)), 6), + "max": round(float(array.max()), 6), + } + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48s_fixed_detector_candidate_worker.py b/experiments/perception/run_m48s_fixed_detector_candidate_worker.py new file mode 100644 index 0000000..cb303cb --- /dev/null +++ b/experiments/perception/run_m48s_fixed_detector_candidate_worker.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 +"""Run one fixed-class detector candidate on the exact M48S risk slice.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path +from typing import Any, Final, Protocol, cast + +import numpy as np +import torch # type: ignore[import-not-found] +from PIL import Image, ImageDraw + +WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0" +EXACT_FRAME_NAMES: Final = ( + "frame-000121.png", + "frame-000131.png", + "frame-000253.png", + "frame-000275.png", + "frame-000443.png", + "frame-000463.png", + "frame-001094.png", + "frame-001228.png", + "frame-001454.png", + "frame-001856.png", + "frame-002386.png", +) +COCO_CLASSES: Final = ( + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "dining table", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +) +AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} +MINIMUM_BOX_AREA_PIXELS: Final = 64.0 +MAXIMUM_BOX_AREA_FRACTION: Final = 0.5 +MINIMUM_VALID_FOV_FRACTION: Final = 0.5 +OVERLAY_THRESHOLD: Final = 0.25 + + +class Detector(Protocol): + def infer(self, image: Image.Image) -> tuple[RawDetection, ...]: ... + + +class RawDetection(tuple[int, str, float, tuple[float, float, float, float]]): + """Normalized detector output: class id, label, score and source-pixel box.""" + + __slots__ = () + + def __new__( + cls, + class_id: int, + label: str, + score: float, + box: tuple[float, float, float, float], + ) -> RawDetection: + return tuple.__new__(cls, (class_id, label, score, box)) + + @property + def class_id(self) -> int: + return self[0] + + @property + def label(self) -> str: + return self[1] + + @property + def score(self) -> float: + return self[2] + + @property + def box(self) -> tuple[float, float, float, float]: + return self[3] + + +class DfineDetector: + """Pinned D-FINE-S COCO PyTorch qualification adapter.""" + + def __init__(self, source_root: Path, config_path: Path, checkpoint: Path) -> None: + sys.path.insert(0, str(source_root)) + from src.core import YAMLConfig # type: ignore[import-not-found] + + config = YAMLConfig(str(config_path), resume=str(checkpoint)) + if "HGNetv2" in config.yaml_cfg: + config.yaml_cfg["HGNetv2"]["pretrained"] = False + state = torch.load(checkpoint, map_location="cpu", weights_only=True) + weights = state["ema"]["module"] if "ema" in state else state["model"] + config.model.load_state_dict(weights) + self._model = config.model.deploy().to("cuda").eval() + self._postprocessor = config.postprocessor.deploy() + + def infer(self, image: Image.Image) -> tuple[RawDetection, ...]: + tensor, ratio, padding = _dfine_preprocess(image) + with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16): + output = self._model(tensor) + size = torch.tensor([[640, 640]], device="cuda") + labels, boxes, scores = self._postprocessor(output, size) + labels_array = labels[0].detach().to("cpu").numpy() + boxes_array = boxes[0].detach().to("cpu").numpy() + scores_array = scores[0].detach().to("cpu").numpy() + pad_x, pad_y = padding + detections = [] + for raw_label, raw_score, raw_box in zip( + labels_array, + scores_array, + boxes_array, + strict=True, + ): + class_id = int(raw_label) + if not 0 <= class_id < len(COCO_CLASSES): + continue + box = ( + (float(raw_box[0]) - pad_x) / ratio, + (float(raw_box[1]) - pad_y) / ratio, + (float(raw_box[2]) - pad_x) / ratio, + (float(raw_box[3]) - pad_y) / ratio, + ) + detections.append( + RawDetection(class_id, COCO_CLASSES[class_id], float(raw_score), box) + ) + return tuple(detections) + + +class RfDetrDetector: + """Pinned RF-DETR-L COCO PyTorch qualification adapter.""" + + def __init__(self, checkpoint: Path) -> None: + from rfdetr import RFDETRLarge # type: ignore[import-not-found] + + self._model = RFDETRLarge(pretrain_weights=str(checkpoint)) + self._model.inference(compile=False, dtype=torch.float16, inplace=True) + + def infer(self, image: Image.Image) -> tuple[RawDetection, ...]: + prediction = self._model.predict( + image, + threshold=0.1, + include_source_image=False, + ) + boxes = np.asarray(prediction.xyxy) + scores = np.asarray(prediction.confidence) + names = np.asarray(prediction.data["class_name"]) + detections = [] + for raw_name, raw_score, raw_box in zip(names, scores, boxes, strict=True): + label = str(raw_name) + try: + class_id = COCO_CLASSES.index(label) + except ValueError: + continue + detections.append( + RawDetection( + class_id, + label, + float(raw_score), + cast( + tuple[float, float, float, float], + tuple(float(value) for value in raw_box), + ), + ) + ) + return tuple(detections) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--candidate", + choices=("dfine-s-coco", "rf-detr-large-coco"), + required=True, + ) + parser.add_argument("--profile-id", required=True) + parser.add_argument("--provider-id", required=True) + parser.add_argument("--upstream-revision", required=True) + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--valid-fov-mask", type=Path, required=True) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--expected-checkpoint-sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--dfine-root", type=Path, default=Path("/opt/dfine")) + parser.add_argument( + "--dfine-config", + type=Path, + default=Path("/opt/dfine/configs/dfine/dfine_hgnetv2_s_coco.yml"), + ) + parser.add_argument("--warmup-iterations", type=int, default=5) + parser.add_argument("--benchmark-iterations", type=int, default=30) + arguments = parser.parse_args() + if arguments.warmup_iterations < 1 or arguments.benchmark_iterations < 1: + raise RuntimeError("warmup and benchmark iterations must be positive") + input_root = arguments.input_root.resolve(strict=True) + images = tuple(sorted(input_root.glob("frame-*.png"))) + if tuple(path.name for path in images) != EXACT_FRAME_NAMES: + raise RuntimeError("candidate Worker requires the exact M48S risk slice") + mask = _load_mask(arguments.valid_fov_mask.resolve(strict=True)) + checkpoint = arguments.checkpoint.resolve(strict=True) + checkpoint_sha256 = _sha256(checkpoint) + if checkpoint_sha256 != arguments.expected_checkpoint_sha256: + raise RuntimeError("checkpoint SHA-256 does not match the pinned profile") + output = arguments.output.absolute() + if output.exists(): + raise RuntimeError("candidate Worker output already exists") + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + overlay_root = output.parent / f"{output.stem}-overlays" + overlay_root.mkdir(mode=0o700) + gpu_before = _gpu_sample() + started_utc_ns = time.time_ns() + torch.cuda.reset_peak_memory_stats() + detector: Detector + if arguments.candidate == "dfine-s-coco": + detector = DfineDetector( + arguments.dfine_root.resolve(strict=True), + arguments.dfine_config.resolve(strict=True), + checkpoint, + ) + else: + detector = RfDetrDetector(checkpoint) + with Image.open(images[0]) as opened: + warmup_image = opened.convert("RGB") + for _ in range(arguments.warmup_iterations): + detector.infer(warmup_image) + _synchronize() + frames = [] + totals: Counter[str] = Counter() + evidence_timings = [] + for image_path in images: + with Image.open(image_path) as opened: + image = opened.convert("RGB") + _synchronize() + started = time.perf_counter_ns() + raw = detector.infer(image) + _synchronize() + completed = time.perf_counter_ns() + elapsed_ms = _milliseconds(started, completed) + evidence_timings.append(elapsed_ms) + detections = _qualify(raw, mask, image.size) + totals["frame_count"] += 1 + totals["detection_count"] += len(detections) + for detection in detections: + totals[f"class:{detection['label']}"] += 1 + frames.append( + { + "frame_name": image_path.name, + "source_sha256": _sha256(image_path), + "detections": detections, + "timing_ms": {"end_to_end": elapsed_ms}, + "authority": AUTHORITY, + } + ) + _write_overlay(image, detections, overlay_root / image_path.name) + with Image.open(input_root / "frame-000253.png") as opened: + benchmark_image = opened.convert("RGB") + benchmark_timings = [] + for _ in range(arguments.benchmark_iterations): + _synchronize() + started = time.perf_counter_ns() + detector.infer(benchmark_image) + _synchronize() + benchmark_timings.append(_milliseconds(started, time.perf_counter_ns())) + gpu_after = _gpu_sample() + timing = _timing_summary(evidence_timings) + benchmark_timing = _timing_summary(benchmark_timings) + metrics = { + "frames": {"requested": 11, "completed": totals["frame_count"]}, + "detection_count_at_minimum_score_0_1": totals["detection_count"], + "class_counts_at_minimum_score_0_1": { + key.removeprefix("class:"): value + for key, value in sorted(totals.items()) + if key.startswith("class:") + }, + "evidence_timing_ms": timing, + "benchmark": { + "frame_name": "frame-000253.png", + "iterations": arguments.benchmark_iterations, + "timing_ms": benchmark_timing, + "core_capacity_fps": round(1000.0 / benchmark_timing["mean"], 6), + }, + "torch_peak_memory": { + "allocated_bytes": torch.cuda.max_memory_allocated(), + "reserved_bytes": torch.cuda.max_memory_reserved(), + }, + "gpu_before": gpu_before, + "gpu_after": gpu_after, + } + document = { + "schema_version": WORKER_RUN_SCHEMA, + "profile_id": arguments.profile_id, + "provider_id": arguments.provider_id, + "candidate": arguments.candidate, + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": checkpoint_sha256, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": totals["frame_count"] == 11, + "execution": { + "worker_id": "worker-006", + "device": torch.cuda.get_device_name(0), + "precision": "fp16", + "inference_passes_per_evidence_frame": 1, + "warmup_iterations": arguments.warmup_iterations, + "benchmark_iterations": arguments.benchmark_iterations, + "concurrent_services_retained": True, + }, + "frames": frames, + "metrics": metrics, + "authority": AUTHORITY, + } + output.write_bytes(_canonical_json(document) + b"\n") + print(output) + print(json.dumps(metrics, indent=2, sort_keys=True)) + return 0 + + +def _dfine_preprocess( + image: Image.Image, +) -> tuple[torch.Tensor, float, tuple[int, int]]: + width, height = image.size + ratio = min(640.0 / width, 640.0 / height) + resized_width = int(width * ratio) + resized_height = int(height * ratio) + resized = image.resize((resized_width, resized_height), Image.Resampling.BILINEAR) + padded = Image.new("RGB", (640, 640)) + pad_x = (640 - resized_width) // 2 + pad_y = (640 - resized_height) // 2 + padded.paste(resized, (pad_x, pad_y)) + array = np.asarray(padded, dtype=np.float32) / 255.0 + tensor = torch.from_numpy(np.ascontiguousarray(array.transpose(2, 0, 1))) + return tensor.unsqueeze(0).to("cuda", non_blocking=True), ratio, (pad_x, pad_y) + + +def _qualify( + detections: tuple[RawDetection, ...], + mask: np.ndarray[Any, Any], + image_size: tuple[int, int], +) -> list[dict[str, object]]: + width, height = image_size + image_area = float(width * height) + qualified = [] + for detection in detections: + if detection.score < 0.1: + continue + x1, y1, x2, y2 = detection.box + x1 = max(0.0, min(float(width), x1)) + y1 = max(0.0, min(float(height), y1)) + x2 = max(0.0, min(float(width), x2)) + y2 = max(0.0, min(float(height), y2)) + area = max(0.0, x2 - x1) * max(0.0, y2 - y1) + if area < MINIMUM_BOX_AREA_PIXELS or area > MAXIMUM_BOX_AREA_FRACTION * image_area: + continue + center_x = min(width - 1, max(0, int((x1 + x2) / 2.0))) + center_y = min(height - 1, max(0, int((y1 + y2) / 2.0))) + if not bool(mask[center_y, center_x]): + continue + ix1 = min(width - 1, max(0, int(np.floor(x1)))) + iy1 = min(height - 1, max(0, int(np.floor(y1)))) + ix2 = min(width, max(ix1 + 1, int(np.ceil(x2)))) + iy2 = min(height, max(iy1 + 1, int(np.ceil(y2)))) + valid_fraction = float(mask[iy1:iy2, ix1:ix2].mean()) + if valid_fraction < MINIMUM_VALID_FOV_FRACTION: + continue + qualified.append( + { + "class_id": detection.class_id, + "label": detection.label, + "score": round(detection.score, 6), + "bbox_xyxy": [round(value, 3) for value in (x1, y1, x2, y2)], + "valid_fov_fraction": round(valid_fraction, 6), + } + ) + qualified.sort(key=lambda item: (-cast(float, item["score"]), cast(int, item["class_id"]))) + return qualified + + +def _write_overlay( + image: Image.Image, + detections: list[dict[str, object]], + destination: Path, +) -> None: + annotated = image.copy() + draw = ImageDraw.Draw(annotated) + for detection in detections: + score = cast(float, detection["score"]) + if score < OVERLAY_THRESHOLD: + continue + box = cast(list[float], detection["bbox_xyxy"]) + label = cast(str, detection["label"]) + draw.rectangle(box, outline=(255, 84, 0), width=3) + draw.text((box[0] + 3, box[1] + 3), f"{label} {score:.2f}", fill=(255, 255, 255)) + annotated.save(destination) + + +def _load_mask(path: Path) -> np.ndarray[Any, Any]: + with Image.open(path) as image: + array = np.asarray(image.convert("L"), dtype=np.uint8) + if array.shape != (600, 800): + raise RuntimeError("valid-FOV mask must be 800x600") + return array > 0 + + +def _gpu_sample() -> dict[str, object]: + completed = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=name,memory.total,memory.used,utilization.gpu,temperature.gpu,power.draw", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + ) + values = [value.strip() for value in completed.stdout.strip().split(",")] + if len(values) != 6: + raise RuntimeError("unexpected nvidia-smi response") + return { + "name": values[0], + "memory_total_mib": float(values[1]), + "memory_used_mib": float(values[2]), + "utilization_gpu_percent": float(values[3]), + "temperature_c": float(values[4]), + "power_w": float(values[5]), + } + + +def _synchronize() -> None: + torch.cuda.synchronize() + + +def _timing_summary(values: list[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "mean": round(float(array.mean()), 6), + "p50": round(float(np.percentile(array, 50)), 6), + "p95": round(float(np.percentile(array, 95)), 6), + "max": round(float(array.max()), 6), + } + + +def _milliseconds(started: int, completed: int) -> float: + return round(max(0, completed - started) / 1_000_000.0, 6) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48s_rf_detr_load_worker.py b/experiments/perception/run_m48s_rf_detr_load_worker.py new file mode 100644 index 0000000..dac2b94 --- /dev/null +++ b/experiments/perception/run_m48s_rf_detr_load_worker.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Run a source-paced bounded-queue RF-DETR/Triton stability qualification.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import resource +import subprocess +import threading +import time +from collections import Counter, deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +import av # type: ignore[import-not-found] +import numpy as np +import run_rf_detr_triton as qualifier # type: ignore[import-not-found] +import tritonclient.http as httpclient # type: ignore[import-not-found] +from PIL import Image + +SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-source-paced-load/v0" +SOURCE_SHA256: Final = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8" +SOURCE_FPS: Final = 10.003944527024467 +AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +@dataclass(frozen=True, slots=True) +class SourceFrame: + sequence: int + scheduled_ns: int + image: Image.Image + + +class LatestWinsQueue: + """Bounded source queue that reports every replacement explicitly.""" + + def __init__(self, capacity: int) -> None: + if capacity < 1: + raise ValueError("queue capacity must be positive") + self.capacity = capacity + self._items: deque[SourceFrame] = deque() + self._condition = threading.Condition() + self._closed = False + self.replacements = 0 + self.maximum_depth = 0 + + def put(self, item: SourceFrame) -> None: + with self._condition: + if self._closed: + return + if len(self._items) == self.capacity: + self._items.popleft() + self.replacements += 1 + self._items.append(item) + self.maximum_depth = max(self.maximum_depth, len(self._items)) + self._condition.notify() + + def get(self) -> SourceFrame | None: + with self._condition: + while not self._items and not self._closed: + self._condition.wait(timeout=1.0) + if self._items: + return self._items.popleft() + return None + + def close(self) -> None: + with self._condition: + self._closed = True + self._condition.notify_all() + + +class GpuTelemetry: + def __init__(self, interval_seconds: float) -> None: + self.interval_seconds = interval_seconds + self.samples: list[dict[str, float]] = [] + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def __enter__(self) -> GpuTelemetry: + self._thread.start() + return self + + def __exit__(self, *_args: object) -> None: + self._stop.set() + self._thread.join(timeout=10.0) + + def _run(self) -> None: + while not self._stop.is_set(): + try: + completed = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=utilization.gpu,memory.used,power.draw,temperature.gpu", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + timeout=10.0, + ) + values = [float(value.strip()) for value in completed.stdout.split(",")] + if len(values) == 4: + self.samples.append( + { + "gpu_utilization_percent": values[0], + "gpu_memory_used_mib": values[1], + "gpu_power_w": values[2], + "gpu_temperature_c": values[3], + } + ) + except (OSError, ValueError, subprocess.SubprocessError): + pass + self._stop.wait(self.interval_seconds) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source-video", type=Path, required=True) + parser.add_argument("--valid-fov-mask", type=Path, required=True) + parser.add_argument("--endpoint", default="localhost:8100") + parser.add_argument("--model-name", default="rf_detr_large") + parser.add_argument("--duration-seconds", type=float, default=1800.0) + parser.add_argument("--queue-capacity", type=int, default=2) + parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--progress", type=Path, required=True) + arguments = parser.parse_args() + if arguments.duration_seconds <= 0: + raise RuntimeError("duration must be positive") + if arguments.telemetry_interval_seconds <= 0: + raise RuntimeError("telemetry interval must be positive") + source = arguments.source_video.resolve(strict=True) + if _sha256(source) != SOURCE_SHA256: + raise RuntimeError("RAVNOVES00 camera stream identity changed") + mask = qualifier._load_mask(arguments.valid_fov_mask.resolve(strict=True)) + output = arguments.output.absolute() + progress = arguments.progress.absolute() + if output.exists() or progress.exists(): + raise RuntimeError("load result or progress artifact already exists") + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + + client = httpclient.InferenceServerClient(arguments.endpoint, concurrency=1) + if not client.is_server_ready() or not client.is_model_ready(arguments.model_name): + raise RuntimeError("isolated Triton or RF-DETR model is not ready") + metadata = client.get_model_metadata(arguments.model_name) + qualifier._validate_metadata(metadata) + triton_before = client.get_inference_statistics(arguments.model_name) + gpu_before = qualifier._gpu_sample() + rss_before_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + queue = LatestWinsQueue(arguments.queue_capacity) + producer_stop = threading.Event() + producer_errors: list[str] = [] + produced_count = [0] + source_loops = [0] + started_ns = time.monotonic_ns() + started_utc_ns = time.time_ns() + producer = threading.Thread( + target=_produce, + args=( + source, + queue, + producer_stop, + producer_errors, + produced_count, + source_loops, + started_ns, + arguments.duration_seconds, + ), + daemon=True, + ) + consumed = 0 + failures = 0 + class_counts: Counter[str] = Counter() + end_to_end_ms: list[float] = [] + triton_round_trip_ms: list[float] = [] + completion_age_ms: list[float] = [] + last_progress_ns = started_ns + producer.start() + with progress.open("x", encoding="utf-8") as progress_stream, GpuTelemetry( + arguments.telemetry_interval_seconds + ) as telemetry: + try: + while True: + item = queue.get() + if item is None: + break + raw, timing = qualifier._infer(client, arguments.model_name, item.image) + detections = qualifier._qualify(raw, mask, item.image.size) + for detection in detections: + if float(detection["score"]) >= 0.5: + class_counts[str(detection["label"])] += 1 + consumed += 1 + end_to_end_ms.append(timing["end_to_end"]) + triton_round_trip_ms.append(timing["triton_round_trip"]) + completion_age_ms.append((time.monotonic_ns() - item.scheduled_ns) / 1_000_000.0) + now_ns = time.monotonic_ns() + if now_ns - last_progress_ns >= 60_000_000_000: + row = { + "elapsed_seconds": round((now_ns - started_ns) / 1_000_000_000.0, 3), + "produced": produced_count[0], + "consumed": consumed, + "replacements": queue.replacements, + "completion_age_p95_ms": _distribution(completion_age_ms)["p95"], + } + progress_stream.write(json.dumps(row, separators=(",", ":")) + "\n") + progress_stream.flush() + print(json.dumps(row, sort_keys=True), flush=True) + last_progress_ns = now_ns + except BaseException: + failures += 1 + raise + finally: + producer_stop.set() + queue.close() + producer.join(timeout=15.0) + + completed_ns = time.monotonic_ns() + wall_seconds = (completed_ns - started_ns) / 1_000_000_000.0 + if producer.is_alive(): + raise RuntimeError("source producer did not stop") + if producer_errors: + raise RuntimeError(f"source producer failed: {producer_errors}") + triton_after = client.get_inference_statistics(arguments.model_name) + gpu_after = qualifier._gpu_sample() + rss_after_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + core = _distribution(end_to_end_ms) + completion_age = _distribution(completion_age_ms) + telemetry_summary = _telemetry_summary(telemetry.samples) + checks = { + "minimum_duration": wall_seconds >= arguments.duration_seconds, + "zero_failures": failures == 0, + "minimum_source_fps": consumed / arguments.duration_seconds >= 9.5, + "maximum_detector_completion_age_p95_ms": completion_age["p95"] <= 175.0, + "maximum_worker_vram_gib": ( + float(telemetry_summary["gpu_memory_used_mib"]["maximum"]) <= 20 * 1024 + ), + "no_sustained_100_percent_gpu": _longest_full_gpu_run(telemetry.samples) + < max(5, round(30.0 / arguments.telemetry_interval_seconds)), + "bounded_latest_wins_queue": queue.maximum_depth <= arguments.queue_capacity, + "triton_request_accounting": _triton_inference_count(triton_after) + - _triton_inference_count(triton_before) + == consumed, + } + detector_load_gate_passed = all(checks.values()) + document = { + "schema_version": SCHEMA_VERSION, + "profile_id": "rf-detr-large-coco-704-trt11-fp16-source-paced/v0", + "source": { + "source_id": "RAVNOVES00", + "sha256": SOURCE_SHA256, + "frame_rate": SOURCE_FPS, + "duration_seconds": arguments.duration_seconds, + "source_loops": source_loops[0], + }, + "model_metadata": metadata, + "execution": { + "worker_id": "worker-006", + "queue_policy": "bounded-latest-wins", + "queue_capacity": arguments.queue_capacity, + "queue_maximum_depth": queue.maximum_depth, + "source_frames_produced": produced_count[0], + "source_frames_consumed": consumed, + "source_frame_replacements": queue.replacements, + "failures": failures, + "wall_seconds": round(wall_seconds, 6), + "effective_consumed_fps": round(consumed / arguments.duration_seconds, 6), + "background_services_retained": True, + }, + "metrics": { + "end_to_end_ms": core, + "triton_round_trip_ms": _distribution(triton_round_trip_ms), + "detector_completion_age_ms": completion_age, + "class_counts_at_score_0_5": dict(sorted(class_counts.items())), + "gpu": telemetry_summary, + "gpu_before": gpu_before, + "gpu_after": gpu_after, + "process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6), + "process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6), + "triton_statistics_before": triton_before, + "triton_statistics_after": triton_after, + }, + "checks": checks, + "detector_load_gate_passed": detector_load_gate_passed, + "integrated_world_state_gate_evaluated": False, + "candidate_accepted": False, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": True, + "authority": AUTHORITY, + } + output.write_bytes(_canonical_json(document) + b"\n") + print(output) + print(json.dumps(document["execution"], indent=2, sort_keys=True)) + print(json.dumps(checks, indent=2, sort_keys=True)) + return 0 if detector_load_gate_passed else 2 + + +def _produce( + source: Path, + queue: LatestWinsQueue, + stop: threading.Event, + errors: list[str], + produced_count: list[int], + source_loops: list[int], + started_ns: int, + duration_seconds: float, +) -> None: + try: + period_ns = round(1_000_000_000.0 / SOURCE_FPS) + while not stop.is_set(): + container = av.open(str(source)) + try: + streams = container.streams.video + if len(streams) != 1: + raise RuntimeError("RAVNOVES00 video stream count changed") + for decoded in container.decode(streams[0]): + sequence = produced_count[0] + scheduled_ns = started_ns + sequence * period_ns + if scheduled_ns - started_ns >= round(duration_seconds * 1_000_000_000): + queue.close() + return + remaining_seconds = (scheduled_ns - time.monotonic_ns()) / 1_000_000_000.0 + if remaining_seconds > 0 and stop.wait(remaining_seconds): + queue.close() + return + bgr = decoded.to_ndarray(format="bgr24") + if bgr.shape != (600, 800, 3): + raise RuntimeError("RAVNOVES00 source raster changed") + rgb = np.ascontiguousarray(bgr[:, :, ::-1]) + queue.put(SourceFrame(sequence, scheduled_ns, Image.fromarray(rgb, "RGB"))) + produced_count[0] += 1 + if stop.is_set(): + queue.close() + return + source_loops[0] += 1 + finally: + container.close() + except BaseException as error: + errors.append(f"{type(error).__name__}: {error}") + queue.close() + + +def _distribution(values: list[float]) -> dict[str, float]: + if not values: + return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "maximum": 0.0} + array = np.asarray(values, dtype=np.float64) + return { + "mean": round(float(array.mean()), 6), + "p50": round(float(np.percentile(array, 50)), 6), + "p95": round(float(np.percentile(array, 95)), 6), + "maximum": round(float(array.max()), 6), + } + + +def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]: + summary: dict[str, Any] = {"sample_count": len(samples)} + for key in ( + "gpu_utilization_percent", + "gpu_memory_used_mib", + "gpu_power_w", + "gpu_temperature_c", + ): + summary[key] = _distribution([sample[key] for sample in samples]) + summary["longest_100_percent_gpu_sample_run"] = _longest_full_gpu_run(samples) + return summary + + +def _longest_full_gpu_run(samples: list[dict[str, float]]) -> int: + longest = 0 + current = 0 + for sample in samples: + if sample["gpu_utilization_percent"] >= 100.0: + current += 1 + longest = max(longest, current) + else: + current = 0 + return longest + + +def _triton_inference_count(statistics: dict[str, Any]) -> int: + model_stats = statistics.get("model_stats") + if not isinstance(model_stats, list) or len(model_stats) != 1: + raise RuntimeError("unexpected Triton model statistics") + count = model_stats[0].get("inference_count") + if not isinstance(count, int): + raise RuntimeError("Triton inference count is unavailable") + return count + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/run_m48s_rf_detr_triton_worker.py b/experiments/perception/run_m48s_rf_detr_triton_worker.py new file mode 100644 index 0000000..37fdc07 --- /dev/null +++ b/experiments/perception/run_m48s_rf_detr_triton_worker.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Qualify the pinned RF-DETR TensorRT finalist through isolated Triton.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import time +from collections import Counter +from pathlib import Path +from typing import Any, Final, cast + +import numpy as np +import tritonclient.http as httpclient # type: ignore[import-not-found] +from PIL import Image, ImageDraw +from torchvision.transforms import functional as vision_functional # type: ignore[import-not-found] + +WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0" +EXACT_FRAME_NAMES: Final = ( + "frame-000121.png", + "frame-000131.png", + "frame-000253.png", + "frame-000275.png", + "frame-000443.png", + "frame-000463.png", + "frame-001094.png", + "frame-001228.png", + "frame-001454.png", + "frame-001856.png", + "frame-002386.png", +) +COCO_CLASSES: Final = ( + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", + "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", + "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", + "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", + "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", + "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", + "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", + "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", +) +COCO_SPARSE_IDS: Final = ( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, +) +COCO_SPARSE_NAMES: Final = dict(zip(COCO_SPARSE_IDS, COCO_CLASSES, strict=True)) +AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} +MEANS: Final = (0.485, 0.456, 0.406) +STDS: Final = (0.229, 0.224, 0.225) +MINIMUM_BOX_AREA_PIXELS: Final = 64.0 +MAXIMUM_BOX_AREA_FRACTION: Final = 0.5 +MINIMUM_VALID_FOV_FRACTION: Final = 0.5 +OVERLAY_THRESHOLD: Final = 0.25 + + +class RawDetection(tuple[int, str, float, tuple[float, float, float, float]]): + __slots__ = () + + def __new__( + cls, + class_id: int, + label: str, + score: float, + box: tuple[float, float, float, float], + ) -> RawDetection: + return tuple.__new__(cls, (class_id, label, score, box)) + + @property + def class_id(self) -> int: + return self[0] + + @property + def label(self) -> str: + return self[1] + + @property + def score(self) -> float: + return self[2] + + @property + def box(self) -> tuple[float, float, float, float]: + return self[3] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", default="localhost:8100") + parser.add_argument("--model-name", default="rf_detr_large") + parser.add_argument("--profile-id", required=True) + parser.add_argument("--provider-id", required=True) + parser.add_argument("--upstream-revision", required=True) + parser.add_argument("--input-root", type=Path, required=True) + parser.add_argument("--valid-fov-mask", type=Path, required=True) + parser.add_argument("--engine", type=Path, required=True) + parser.add_argument("--expected-engine-sha256", required=True) + parser.add_argument("--checkpoint-sha256", required=True) + parser.add_argument("--pytorch-reference", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--warmup-iterations", type=int, default=10) + parser.add_argument("--benchmark-iterations", type=int, default=100) + arguments = parser.parse_args() + if arguments.warmup_iterations < 1 or arguments.benchmark_iterations < 1: + raise RuntimeError("warmup and benchmark iterations must be positive") + input_root = arguments.input_root.resolve(strict=True) + images = tuple(sorted(input_root.glob("frame-*.png"))) + if tuple(path.name for path in images) != EXACT_FRAME_NAMES: + raise RuntimeError("Triton qualifier requires the exact M48S risk slice") + mask = _load_mask(arguments.valid_fov_mask.resolve(strict=True)) + engine = arguments.engine.resolve(strict=True) + engine_sha256 = _sha256(engine) + if engine_sha256 != arguments.expected_engine_sha256: + raise RuntimeError("TensorRT engine SHA-256 does not match the pinned finalist") + reference_path = arguments.pytorch_reference.resolve(strict=True) + output = arguments.output.absolute() + if output.exists(): + raise RuntimeError("Triton Worker output already exists") + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + overlay_root = output.parent / f"{output.stem}-overlays" + overlay_root.mkdir(mode=0o700) + + client = httpclient.InferenceServerClient(arguments.endpoint, concurrency=1) + if not client.is_server_ready() or not client.is_model_ready(arguments.model_name): + raise RuntimeError("isolated Triton or RF-DETR model is not ready") + metadata = client.get_model_metadata(arguments.model_name) + _validate_metadata(metadata) + statistics_before = client.get_inference_statistics(arguments.model_name) + gpu_before = _gpu_sample() + started_utc_ns = time.time_ns() + with Image.open(images[0]) as opened: + warmup_image = opened.convert("RGB") + for _ in range(arguments.warmup_iterations): + _infer(client, arguments.model_name, warmup_image) + + frames = [] + totals: Counter[str] = Counter() + evidence_timings = [] + network_timings = [] + for image_path in images: + with Image.open(image_path) as opened: + image = opened.convert("RGB") + raw, timing = _infer(client, arguments.model_name, image) + evidence_timings.append(timing["end_to_end"]) + network_timings.append(timing["triton_round_trip"]) + detections = _qualify(raw, mask, image.size) + totals["frame_count"] += 1 + totals["detection_count"] += len(detections) + for detection in detections: + totals[f"class:{detection['label']}"] += 1 + frames.append( + { + "frame_name": image_path.name, + "source_sha256": _sha256(image_path), + "detections": detections, + "timing_ms": timing, + "authority": AUTHORITY, + } + ) + _write_overlay(image, detections, overlay_root / image_path.name) + + with Image.open(input_root / "frame-000253.png") as opened: + benchmark_image = opened.convert("RGB") + benchmark_timings = [] + benchmark_network_timings = [] + for _ in range(arguments.benchmark_iterations): + _, timing = _infer(client, arguments.model_name, benchmark_image) + benchmark_timings.append(timing["end_to_end"]) + benchmark_network_timings.append(timing["triton_round_trip"]) + gpu_after = _gpu_sample() + statistics_after = client.get_inference_statistics(arguments.model_name) + benchmark = _timing_summary(benchmark_timings) + parity = _reference_parity(frames, reference_path) + metrics = { + "frames": {"requested": 11, "completed": totals["frame_count"]}, + "detection_count_at_minimum_score_0_1": totals["detection_count"], + "class_counts_at_minimum_score_0_1": { + key.removeprefix("class:"): value + for key, value in sorted(totals.items()) + if key.startswith("class:") + }, + "evidence_timing_ms": _timing_summary(evidence_timings), + "evidence_triton_round_trip_ms": _timing_summary(network_timings), + "benchmark": { + "frame_name": "frame-000253.png", + "iterations": arguments.benchmark_iterations, + "timing_ms": benchmark, + "triton_round_trip_ms": _timing_summary(benchmark_network_timings), + "end_to_end_capacity_fps": round(1000.0 / benchmark["mean"], 6), + }, + "pytorch_reference_parity": parity, + "triton_statistics_before": statistics_before, + "triton_statistics_after": statistics_after, + "gpu_before": gpu_before, + "gpu_after": gpu_after, + } + document = { + "schema_version": WORKER_RUN_SCHEMA, + "profile_id": arguments.profile_id, + "provider_id": arguments.provider_id, + "candidate": "rf-detr-large-coco-tensorrt", + "upstream_revision": arguments.upstream_revision, + "checkpoint_sha256": arguments.checkpoint_sha256, + "engine_sha256": engine_sha256, + "started_utc_ns": started_utc_ns, + "completed_utc_ns": time.time_ns(), + "completed": totals["frame_count"] == 11, + "execution": { + "worker_id": "worker-006", + "device": gpu_after["name"], + "precision": "strongly-typed-fp16", + "inference_passes_per_evidence_frame": 1, + "warmup_iterations": arguments.warmup_iterations, + "benchmark_iterations": arguments.benchmark_iterations, + "isolated_triton": True, + "concurrent_services_retained": True, + }, + "model_metadata": metadata, + "frames": frames, + "metrics": metrics, + "authority": AUTHORITY, + } + output.write_bytes(_canonical_json(document) + b"\n") + print(output) + print(json.dumps(metrics, indent=2, sort_keys=True)) + return 0 + + +def _infer( + client: httpclient.InferenceServerClient, + model_name: str, + image: Image.Image, +) -> tuple[tuple[RawDetection, ...], dict[str, float]]: + started = time.perf_counter_ns() + tensor = vision_functional.to_tensor(image) + tensor = vision_functional.resize(tensor, [704, 704], antialias=False) + tensor = vision_functional.normalize(tensor, MEANS, STDS) + batch = np.ascontiguousarray(tensor.unsqueeze(0).numpy(), dtype=np.float32) + preprocessed = time.perf_counter_ns() + infer_input = httpclient.InferInput("input", batch.shape, "FP32") + infer_input.set_data_from_numpy(batch, binary_data=True) + response = client.infer( + model_name, + [infer_input], + outputs=[ + httpclient.InferRequestedOutput("dets", binary_data=True), + httpclient.InferRequestedOutput("labels", binary_data=True), + ], + ) + inferred = time.perf_counter_ns() + boxes = response.as_numpy("dets") + logits = response.as_numpy("labels") + if boxes is None or logits is None: + raise RuntimeError("Triton RF-DETR response is missing outputs") + detections = decode_outputs(boxes, logits, image.size) + completed = time.perf_counter_ns() + return detections, { + "preprocess": _milliseconds(started, preprocessed), + "triton_round_trip": _milliseconds(preprocessed, inferred), + "postprocess": _milliseconds(inferred, completed), + "end_to_end": _milliseconds(started, completed), + } + + +def decode_outputs( + boxes: np.ndarray[Any, Any], + logits: np.ndarray[Any, Any], + image_size: tuple[int, int], +) -> tuple[RawDetection, ...]: + if boxes.shape != (1, 300, 4) or logits.shape != (1, 300, 91): + raise RuntimeError(f"unexpected RF-DETR output shapes: {boxes.shape}, {logits.shape}") + probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits[0].astype(np.float32), -80.0, 80.0))) + flattened = probabilities.reshape(-1) + topk = np.argsort(-flattened, kind="stable")[:300] + width, height = image_size + decoded = [] + for flat_index in topk: + score = float(flattened[flat_index]) + if score <= 0.1: + continue + query_index = int(flat_index // logits.shape[2]) + sparse_class_id = int(flat_index % logits.shape[2]) + label = COCO_SPARSE_NAMES.get(sparse_class_id) + if label is None: + continue + center_x, center_y, box_width, box_height = ( + float(value) for value in boxes[0, query_index].astype(np.float32) + ) + box = ( + (center_x - box_width / 2.0) * width, + (center_y - box_height / 2.0) * height, + (center_x + box_width / 2.0) * width, + (center_y + box_height / 2.0) * height, + ) + decoded.append(RawDetection(COCO_CLASSES.index(label), label, score, box)) + return tuple(decoded) + + +def _qualify( + detections: tuple[RawDetection, ...], + mask: np.ndarray[Any, Any], + image_size: tuple[int, int], +) -> list[dict[str, object]]: + width, height = image_size + image_area = float(width * height) + qualified = [] + for detection in detections: + x1, y1, x2, y2 = detection.box + x1 = max(0.0, min(float(width), x1)) + y1 = max(0.0, min(float(height), y1)) + x2 = max(0.0, min(float(width), x2)) + y2 = max(0.0, min(float(height), y2)) + area = max(0.0, x2 - x1) * max(0.0, y2 - y1) + if area < MINIMUM_BOX_AREA_PIXELS or area > MAXIMUM_BOX_AREA_FRACTION * image_area: + continue + center_x = min(width - 1, max(0, int((x1 + x2) / 2.0))) + center_y = min(height - 1, max(0, int((y1 + y2) / 2.0))) + if not bool(mask[center_y, center_x]): + continue + ix1 = min(width - 1, max(0, int(np.floor(x1)))) + iy1 = min(height - 1, max(0, int(np.floor(y1)))) + ix2 = min(width, max(ix1 + 1, int(np.ceil(x2)))) + iy2 = min(height, max(iy1 + 1, int(np.ceil(y2)))) + valid_fraction = float(mask[iy1:iy2, ix1:ix2].mean()) + if valid_fraction < MINIMUM_VALID_FOV_FRACTION: + continue + qualified.append( + { + "class_id": detection.class_id, + "label": detection.label, + "score": round(detection.score, 6), + "bbox_xyxy": [round(value, 3) for value in (x1, y1, x2, y2)], + "valid_fov_fraction": round(valid_fraction, 6), + } + ) + qualified.sort(key=lambda item: (-cast(float, item["score"]), cast(int, item["class_id"]))) + return qualified + + +def _reference_parity(frames: list[dict[str, object]], reference_path: Path) -> dict[str, object]: + reference = json.loads(reference_path.read_text(encoding="utf-8")) + if not isinstance(reference, dict): + raise RuntimeError("PyTorch reference must be a JSON object") + reference_frames = reference.get("frames") + if not isinstance(reference_frames, list): + raise RuntimeError("PyTorch reference lacks frames") + actual = _best_detection(frames, "frame-000253.png", "dog") + expected = _best_detection(reference_frames, "frame-000253.png", "dog") + if actual is None or expected is None: + return { + "frame_000253_dog_present_in_pytorch": expected is not None, + "frame_000253_dog_present_in_tensorrt": actual is not None, + "score_absolute_delta": None, + "box_iou": None, + "passed": False, + } + score_delta = abs( + float(cast(float, actual["score"])) - float(cast(float, expected["score"])) + ) + iou = _box_iou( + cast(list[float], actual["bbox_xyxy"]), + cast(list[float], expected["bbox_xyxy"]), + ) + return { + "frame_000253_dog_present_in_pytorch": True, + "frame_000253_dog_present_in_tensorrt": True, + "pytorch_score": expected["score"], + "tensorrt_score": actual["score"], + "score_absolute_delta": round(score_delta, 6), + "box_iou": round(iou, 6), + "passed": score_delta <= 0.05 and iou >= 0.9, + } + + +def _best_detection( + frames: list[Any], frame_name: str, label: str +) -> dict[str, object] | None: + for frame in frames: + if not isinstance(frame, dict) or frame.get("frame_name") != frame_name: + continue + detections = frame.get("detections") + if not isinstance(detections, list): + raise RuntimeError("reference frame detections must be a list") + selected = [ + item + for item in detections + if isinstance(item, dict) and item.get("label") == label + ] + return max(selected, key=lambda item: float(item["score"])) if selected else None + raise RuntimeError(f"reference frame not found: {frame_name}") + + +def _box_iou(left: list[float], right: list[float]) -> float: + intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0])) + intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1])) + intersection = intersection_width * intersection_height + left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1]) + right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1]) + union = left_area + right_area - intersection + return intersection / union if union > 0 else 0.0 + + +def _validate_metadata(metadata: dict[str, Any]) -> None: + expected_inputs = [{"name": "input", "datatype": "FP32", "shape": [1, 3, 704, 704]}] + expected_outputs = [ + {"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]}, + {"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]}, + ] + if metadata.get("inputs") != expected_inputs or metadata.get("outputs") != expected_outputs: + raise RuntimeError(f"unexpected isolated Triton model metadata: {metadata}") + + +def _write_overlay( + image: Image.Image, detections: list[dict[str, object]], destination: Path +) -> None: + annotated = image.copy() + draw = ImageDraw.Draw(annotated) + for detection in detections: + score = cast(float, detection["score"]) + if score < OVERLAY_THRESHOLD: + continue + box = cast(list[float], detection["bbox_xyxy"]) + draw.rectangle(box, outline=(0, 220, 112), width=3) + draw.text( + (box[0] + 3, box[1] + 3), + f"{detection['label']} {score:.2f}", + fill=(255, 255, 255), + ) + annotated.save(destination) + + +def _load_mask(path: Path) -> np.ndarray[Any, Any]: + with Image.open(path) as image: + array = np.asarray(image.convert("L"), dtype=np.uint8) + if array.shape != (600, 800): + raise RuntimeError("valid-FOV mask must be 800x600") + return array > 0 + + +def _gpu_sample() -> dict[str, object]: + completed = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=name,memory.total,memory.used,utilization.gpu,temperature.gpu,power.draw", + "--format=csv,noheader,nounits", + ], + check=True, + capture_output=True, + text=True, + ) + values = [value.strip() for value in completed.stdout.strip().split(",")] + if len(values) != 6: + raise RuntimeError("unexpected nvidia-smi response") + return { + "name": values[0], + "memory_total_mib": float(values[1]), + "memory_used_mib": float(values[2]), + "utilization_gpu_percent": float(values[3]), + "temperature_c": float(values[4]), + "power_w": float(values[5]), + } + + +def _timing_summary(values: list[float]) -> dict[str, float]: + array = np.asarray(values, dtype=np.float64) + return { + "mean": round(float(array.mean()), 6), + "p50": round(float(np.percentile(array, 50)), 6), + "p95": round(float(np.percentile(array, 95)), 6), + "max": round(float(array.max()), 6), + } + + +def _milliseconds(started: int, completed: int) -> float: + return round(max(0, completed - started) / 1_000_000.0, 6) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/seal_m48s_fixed_detector_tournament.py b/experiments/perception/seal_m48s_fixed_detector_tournament.py new file mode 100644 index 0000000..368f5da --- /dev/null +++ b/experiments/perception/seal_m48s_fixed_detector_tournament.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Seal the bounded M48S fixed-class detector tournament.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Final + +from k1link.perception.fixed_class_detector_tournament import ( + TOURNAMENT_SCHEMA, + CandidateWorkerRun, + canonical_json, + false_authority, + sha256_path, +) + +RESULT_PREFIX: Final = "m48s-fixed-detector-tournament-" +THRESHOLDS: Final = (0.25, 0.5) + + +def main() -> int: + repository = Path(__file__).resolve().parents[2] + runtime = repository / ".runtime/compute-experiments/m48s-semantic-shadow" + parser = argparse.ArgumentParser() + parser.add_argument( + "--profile", + type=Path, + default=repository / "config/perception/fixed-class-detector-tournament-v0.json", + ) + parser.add_argument( + "--dfine-result", + type=Path, + default=runtime / "fixed-detector-tournament-worker/dfine-s-worker.json", + ) + parser.add_argument( + "--rf-detr-result", + type=Path, + default=runtime / "fixed-detector-tournament-worker/rf-detr-large-worker.json", + ) + parser.add_argument( + "--yolox-result", + type=Path, + default=( + runtime + / "yolox-all-coco-results" + / ( + "m48s-yolox-all-coco-shadow-" + "7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06" + ) + ), + ) + parser.add_argument( + "--output-root", + type=Path, + default=runtime / "fixed-detector-tournament-results", + ) + arguments = parser.parse_args() + profile_path = arguments.profile.resolve(strict=True) + profile = _load_object(profile_path) + if profile.get("schema_version") != "missioncore.fixed-class-detector-tournament-profile/v0": + raise RuntimeError("unexpected fixed-class tournament profile schema") + dfine_path = arguments.dfine_result.resolve(strict=True) + rf_detr_path = arguments.rf_detr_result.resolve(strict=True) + dfine = CandidateWorkerRun.from_path(dfine_path) + rf_detr = CandidateWorkerRun.from_path(rf_detr_path) + yolox_root = arguments.yolox_result.resolve(strict=True) + yolox_manifest_path = yolox_root / "manifest.json" + yolox_frames_path = yolox_root / "frames.jsonl" + yolox_manifest = _load_object(yolox_manifest_path) + yolox_frames = _load_jsonl(yolox_frames_path) + if yolox_manifest.get("result_id") != profile["source"]["baseline_result_id"]: + raise RuntimeError("YOLOX baseline result does not match the tournament profile") + worker_paths = { + dfine.profile_id: dfine_path, + rf_detr.profile_id: rf_detr_path, + } + candidate_summaries = { + dfine.profile_id: _candidate_summary(dfine), + rf_detr.profile_id: _candidate_summary(rf_detr), + } + baseline_summary = { + "profile_id": "yolox-s-raw-kb4-all-coco-shadow/v2", + "provider_id": "triton-yolox-s-raw-kb4-all-coco/v2", + "quality": { + str(threshold): _yolox_quality(yolox_frames, threshold) + for threshold in THRESHOLDS + }, + "worker_metrics": yolox_manifest["metrics"], + } + evidence = { + "profile_sha256": sha256_path(profile_path), + "worker_result_sha256": { + profile_id: sha256_path(path) for profile_id, path in sorted(worker_paths.items()) + }, + "yolox_manifest_sha256": sha256_path(yolox_manifest_path), + "yolox_frames_sha256": sha256_path(yolox_frames_path), + "manual_visual_review": { + "reviewed_frames": [253, 275, 443, 1228], + "rf_detr_frame_253_dog_box_correct": True, + "dfine_frame_253_dog_box_present_at_0_25": False, + "dfine_observed_confusions": [ + "dog-as-skateboard", + "scanner-body-as-surfboard", + "duplicate-risk-labels-on-one-object", + ], + "rf_detr_observed_advantage": "correct dog and cleaner person/vehicle labeling", + "ground_truth": False, + }, + } + decision = { + "finalist_profile_id": rf_detr.profile_id, + "finalist_provider_id": rf_detr.provider_id, + "eliminated_profile_ids": [ + "yolox-s-raw-kb4-all-coco-shadow/v2", + dfine.profile_id, + ], + "reasons": { + "yolox-s-raw-kb4-all-coco-shadow/v2": ( + "visible frame-253 dog missed; retained only as regression baseline" + ), + dfine.profile_id: ( + "frame-253 dog missed at 0.25 and 0.5; more risk-class confusions; slower qualifier" + ), + rf_detr.profile_id: ( + "correct frame-253 dog at 0.741; cleaner risk labels; 44.786 FPS PyTorch qualifier" + ), + }, + "candidate_accepted": False, + "next_gate": ( + "RF-DETR-L TensorRT FP16 through isolated Triton, then full recorded " + "concurrent-load replay" + ), + } + identity = { + "schema_version": TOURNAMENT_SCHEMA, + "profile_id": profile["profile_id"], + "evidence": evidence, + "baseline": baseline_summary, + "candidates": candidate_summaries, + "decision": decision, + "completed": True, + "accepted": False, + "authority": false_authority(), + } + result_id = RESULT_PREFIX + hashlib.sha256(canonical_json(identity)).hexdigest() + output_root = arguments.output_root.absolute() + output_root.mkdir(mode=0o700, parents=True, exist_ok=True) + destination = output_root / result_id + if destination.exists(): + raise RuntimeError("immutable fixed-class tournament result already exists") + destination.mkdir(mode=0o700) + for source, name in ( + (dfine_path, "dfine-s-worker.json"), + (rf_detr_path, "rf-detr-large-worker.json"), + ): + (destination / name).write_bytes(source.read_bytes()) + manifest = {"result_id": result_id, **identity} + (destination / "manifest.json").write_bytes(canonical_json(manifest) + b"\n") + (destination / "report.json").write_bytes( + canonical_json( + { + "schema_version": TOURNAMENT_SCHEMA, + "result_id": result_id, + "completed": True, + "accepted": False, + "baseline": baseline_summary, + "candidates": candidate_summaries, + "decision": decision, + "authority": false_authority(), + } + ) + + b"\n" + ) + print(result_id) + print(json.dumps(decision, indent=2, sort_keys=True)) + return 0 + + +def _candidate_summary(run: CandidateWorkerRun) -> dict[str, object]: + return { + "profile_id": run.profile_id, + "provider_id": run.provider_id, + "upstream_revision": run.upstream_revision, + "checkpoint_sha256": run.checkpoint_sha256, + "quality": { + str(threshold): run.quality_summary(threshold=threshold) + for threshold in THRESHOLDS + }, + "worker_metrics": run.metrics, + } + + +def _yolox_quality(frames: list[Mapping[str, Any]], threshold: float) -> dict[str, object]: + selected = [] + dog_selected = [] + for frame in frames: + detections = frame.get("all_coco_detections") + if not isinstance(detections, list): + raise RuntimeError("YOLOX frame lacks all-COCO detections") + for detection in detections: + if not isinstance(detection, dict): + raise RuntimeError("YOLOX detection must be an object") + score = detection.get("score") + label = detection.get("label") + if isinstance(score, int | float) and score >= threshold and isinstance(label, str): + selected.append(label) + if frame.get("frame_name") == "frame-000253.jpg" and label == "dog": + dog_selected.append(float(score)) + counts = Counter(selected) + return { + "threshold": threshold, + "detection_count": len(selected), + "class_counts": dict(sorted(counts.items())), + "frame_000253_dog_detected": bool(dog_selected), + "frame_000253_dog_max_score": max(dog_selected) if dog_selected else None, + } + + +def _load_object(path: Path) -> Mapping[str, Any]: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise RuntimeError(f"JSON document must be an object: {path}") + return document + + +def _load_jsonl(path: Path) -> list[Mapping[str, Any]]: + rows: list[Mapping[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + document = json.loads(line) + if not isinstance(document, dict): + raise RuntimeError(f"JSONL row must be an object: {path}") + rows.append(document) + return rows + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/seal_m48s_rf_detr_deployment.py b/experiments/perception/seal_m48s_rf_detr_deployment.py new file mode 100644 index 0000000..7f1d683 --- /dev/null +++ b/experiments/perception/seal_m48s_rf_detr_deployment.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Seal the RF-DETR TensorRT/Triton detector deployment gate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any, Final + +from k1link.perception.fixed_class_detector_tournament import ( + CandidateWorkerRun, + canonical_json, + false_authority, + sha256_path, +) + +SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-deployment-gate/v0" +RESULT_PREFIX: Final = "m48s-rf-detr-deployment-gate-" +ENGINE_SHA256: Final = "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8" +EXPORTED_ONNX_SHA256: Final = ( + "9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300" +) +FP16_ONNX_SHA256: Final = "9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb" + + +def main() -> int: + repository = Path(__file__).resolve().parents[2] + runtime = repository / ".runtime/compute-experiments/m48s-semantic-shadow" + parser = argparse.ArgumentParser() + parser.add_argument( + "--profile", + type=Path, + default=repository / "config/perception/rf-detr-large-risk-shadow-v0.json", + ) + parser.add_argument( + "--tournament", + type=Path, + default=( + runtime + / "fixed-detector-tournament-results" + / ( + "m48s-fixed-detector-tournament-" + "0e61d75e6dc575d53e4bb98772a41d240fe627ad642de5178beb1154636e1299" + ) + / "manifest.json" + ), + ) + parser.add_argument( + "--pytorch-result", + type=Path, + default=runtime / "fixed-detector-tournament-worker/rf-detr-large-worker.json", + ) + parser.add_argument( + "--triton-result", + type=Path, + default=runtime / "fixed-detector-tournament-worker/rf-detr-large-triton-worker.json", + ) + parser.add_argument( + "--onnx-export", + type=Path, + default=runtime / "rf-detr-deployment-worker/rf-detr-large-onnx-export.json", + ) + parser.add_argument( + "--fp16-conversion", + type=Path, + default=( + runtime + / "rf-detr-deployment-worker/rf-detr-large-onnx-fp16-conversion-v4.json" + ), + ) + parser.add_argument( + "--trtexec-log", + type=Path, + default=runtime / "rf-detr-deployment-worker/rf-detr-large-trtexec-build-v6.log", + ) + parser.add_argument( + "--load-result", + type=Path, + default=runtime / "rf-detr-deployment-worker/rf-detr-load-30m.json", + ) + parser.add_argument( + "--output-root", + type=Path, + default=runtime / "rf-detr-deployment-results", + ) + arguments = parser.parse_args() + + paths = { + "profile": arguments.profile.resolve(strict=True), + "tournament": arguments.tournament.resolve(strict=True), + "pytorch_result": arguments.pytorch_result.resolve(strict=True), + "triton_result": arguments.triton_result.resolve(strict=True), + "onnx_export": arguments.onnx_export.resolve(strict=True), + "fp16_conversion": arguments.fp16_conversion.resolve(strict=True), + "trtexec_log": arguments.trtexec_log.resolve(strict=True), + "load_result": arguments.load_result.resolve(strict=True), + } + profile = _load_object(paths["profile"]) + tournament = _load_object(paths["tournament"]) + pytorch = CandidateWorkerRun.from_path(paths["pytorch_result"]) + triton = CandidateWorkerRun.from_path(paths["triton_result"]) + triton_document = _load_object(paths["triton_result"]) + onnx_export = _load_object(paths["onnx_export"]) + fp16_conversion = _load_object(paths["fp16_conversion"]) + load_result = _load_object(paths["load_result"]) + build_log = paths["trtexec_log"].read_text("utf-8") + + _validate( + profile=profile, + tournament=tournament, + pytorch=pytorch, + triton=triton, + triton_document=triton_document, + onnx_export=onnx_export, + fp16_conversion=fp16_conversion, + load_result=load_result, + build_log=build_log, + ) + pytorch_quality = pytorch.quality_summary(threshold=0.5) + triton_quality = triton.quality_summary(threshold=0.5) + evidence = { + "files": { + name: {"sha256": sha256_path(path), "size_bytes": path.stat().st_size} + for name, path in sorted(paths.items()) + }, + "engine_sha256": ENGINE_SHA256, + "exported_onnx_sha256": EXPORTED_ONNX_SHA256, + "strongly_typed_fp16_onnx_sha256": FP16_ONNX_SHA256, + "pytorch_quality_at_0_5": pytorch_quality, + "triton_quality_at_0_5": triton_quality, + "tensorrt_parity": triton_document["metrics"]["pytorch_reference_parity"], + "triton_benchmark": triton_document["metrics"]["benchmark"], + "source_paced_load": { + "execution": load_result["execution"], + "checks": load_result["checks"], + "end_to_end_ms": load_result["metrics"]["end_to_end_ms"], + "detector_completion_age_ms": load_result["metrics"][ + "detector_completion_age_ms" + ], + "gpu": load_result["metrics"]["gpu"], + }, + } + decision = { + "tournament_finalist": True, + "tensorrt_numeric_parity_passed": True, + "detector_source_paced_load_gate_passed": True, + "ready_for_reference_graph_shadow": True, + "integrated_world_state_gate_evaluated": False, + "production_accepted": False, + "next_gate": ( + "run the RF-DETR shadow provider inside the complete reference graph and require " + "world-state p95 <= 175 ms without changing false authority" + ), + } + identity = { + "schema_version": SCHEMA_VERSION, + "profile_id": profile["profile_id"], + "evidence": evidence, + "decision": decision, + "completed": True, + "accepted": False, + "authority": false_authority(), + } + result_id = RESULT_PREFIX + hashlib.sha256(canonical_json(identity)).hexdigest() + destination = arguments.output_root.absolute() / result_id + if destination.exists(): + raise RuntimeError("immutable RF-DETR deployment result already exists") + destination.mkdir(mode=0o700, parents=True) + for name, path in paths.items(): + suffix = path.suffix or ".evidence" + (destination / f"{name}{suffix}").write_bytes(path.read_bytes()) + manifest = {"result_id": result_id, **identity} + (destination / "manifest.json").write_bytes(canonical_json(manifest) + b"\n") + (destination / "report.json").write_bytes( + canonical_json( + { + "schema_version": SCHEMA_VERSION, + "result_id": result_id, + "completed": True, + "accepted": False, + "evidence": evidence, + "decision": decision, + "authority": false_authority(), + } + ) + + b"\n" + ) + print(result_id) + print(json.dumps(decision, indent=2, sort_keys=True)) + return 0 + + +def _validate( + *, + profile: dict[str, Any], + tournament: dict[str, Any], + pytorch: CandidateWorkerRun, + triton: CandidateWorkerRun, + triton_document: dict[str, Any], + onnx_export: dict[str, Any], + fp16_conversion: dict[str, Any], + load_result: dict[str, Any], + build_log: str, +) -> None: + if profile.get("schema_version") != "missioncore.rf-detr-risk-shadow-profile/v0": + raise RuntimeError("unexpected RF-DETR shadow profile schema") + decision = tournament.get("decision") + if not isinstance(decision, dict) or decision.get("finalist_profile_id") != pytorch.profile_id: + raise RuntimeError("tournament does not select the RF-DETR PyTorch reference") + if triton.profile_id != "rf-detr-large-coco-704-trt11-fp16/v0": + raise RuntimeError("unexpected RF-DETR Triton profile") + if triton_document.get("engine_sha256") != ENGINE_SHA256: + raise RuntimeError("RF-DETR Triton engine identity changed") + parity = triton_document.get("metrics", {}).get("pytorch_reference_parity", {}) + if not isinstance(parity, dict) or parity.get("passed") is not True: + raise RuntimeError("RF-DETR TensorRT numeric parity failed") + if pytorch.quality_summary(threshold=0.5)["class_counts"] != triton.quality_summary( + threshold=0.5 + )["class_counts"]: + raise RuntimeError("RF-DETR TensorRT 0.5 class counts diverged from PyTorch") + if onnx_export.get("onnx", {}).get("sha256") != EXPORTED_ONNX_SHA256: + raise RuntimeError("RF-DETR exported ONNX identity changed") + if fp16_conversion.get("output_onnx_sha256") != FP16_ONNX_SHA256: + raise RuntimeError("RF-DETR strongly typed FP16 ONNX identity changed") + required_build_markers = ( + "Precision: Strongly Typed", + "Input binding for input with dimensions 1x3x704x704 and type fp32", + "Output binding for dets with dimensions 1x300x4 and type fp16", + "Output binding for labels with dimensions 1x300x91 and type fp16", + "&&&& PASSED TensorRT.trtexec", + ) + if any(marker not in build_log for marker in required_build_markers): + raise RuntimeError("TensorRT build log is incomplete") + if ( + load_result.get("schema_version") + != "missioncore.m48s-rf-detr-source-paced-load/v0" + or load_result.get("completed") is not True + or load_result.get("detector_load_gate_passed") is not True + or load_result.get("candidate_accepted") is not False + or load_result.get("integrated_world_state_gate_evaluated") is not False + ): + raise RuntimeError("RF-DETR source-paced load result is incompatible") + checks = load_result.get("checks") + if ( + not isinstance(checks, dict) + or not checks + or not all(value is True for value in checks.values()) + ): + raise RuntimeError("RF-DETR source-paced load checks did not all pass") + if load_result.get("authority") != false_authority(): + raise RuntimeError("RF-DETR source-paced load gained authority") + + +def _load_object(path: Path) -> dict[str, Any]: + document = json.loads(path.read_text("utf-8")) + if not isinstance(document, dict): + raise RuntimeError(f"JSON document must be an object: {path}") + return document + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/Dockerfile.fixed-detector-tournament b/experiments/perception/worker/Dockerfile.fixed-detector-tournament new file mode 100644 index 0000000..11bade0 --- /dev/null +++ b/experiments/perception/worker/Dockerfile.fixed-detector-tournament @@ -0,0 +1,30 @@ +FROM nvcr.io/nvidia/tritonserver:26.06-py3 + +ARG DFINE_REVISION=956d1709314c2c6a4df6f34de232054578a7449f + +RUN python3 -m pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu130 \ + "torch==2.9.1+cu130" \ + "torchvision==0.24.1+cu130" + +RUN git clone https://github.com/Peterande/D-FINE.git /opt/dfine \ + && git -C /opt/dfine checkout --detach "${DFINE_REVISION}" \ + && test "$(git -C /opt/dfine rev-parse HEAD)" = "${DFINE_REVISION}" + +RUN python3 -m pip install --no-cache-dir \ + "rfdetr[onnx]==1.9.4" \ + "numpy==1.26.4" \ + "ml_dtypes==0.5.4" \ + "onnxconverter-common==1.16.0" \ + "tritonclient[http]==2.71.0" \ + "faster-coco-eval>=1.6.6" \ + "PyYAML>=6.0" \ + "scipy>=1.10" \ + "calflops>=0.3" \ + "loguru>=0.7" \ + "tensorboard>=2.17" + +LABEL com.nodedc.product="mission-core" \ + com.nodedc.stack="ndc-mission-core-compute" \ + com.nodedc.role="bounded-detector-qualification" \ + com.nodedc.managed-by="codex-bounded-experiment" diff --git a/experiments/perception/worker/bootstrap_fixed_detector_tournament.sh b/experiments/perception/worker/bootstrap_fixed_detector_tournament.sh new file mode 100644 index 0000000..672263d --- /dev/null +++ b/experiments/perception/worker/bootstrap_fixed_detector_tournament.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly DFINE_REVISION="956d1709314c2c6a4df6f34de232054578a7449f" + +python3 -m pip install --no-cache-dir \ + --index-url https://download.pytorch.org/whl/cu130 \ + "torch==2.9.1+cu130" \ + "torchvision==0.24.1+cu130" + +git clone https://github.com/Peterande/D-FINE.git /opt/dfine +git -C /opt/dfine checkout --detach "${DFINE_REVISION}" +test "$(git -C /opt/dfine rev-parse HEAD)" = "${DFINE_REVISION}" + +python3 -m pip install --no-cache-dir \ + "rfdetr[onnx]==1.9.4" \ + "numpy==1.26.4" \ + "ml_dtypes==0.5.4" \ + "onnxconverter-common==1.16.0" \ + "tritonclient[http]==2.71.0" \ + "faster-coco-eval>=1.6.6" \ + "PyYAML>=6.0" \ + "scipy>=1.10" \ + "calflops>=0.3" \ + "loguru>=0.7" \ + "tensorboard>=2.17" + +python3 -c "import importlib.metadata, rfdetr, torch, torchvision; print(torch.__version__, torchvision.__version__, importlib.metadata.version('rfdetr'))" diff --git a/experiments/perception/worker/build_m48s_rf_detr_tensorrt.sh b/experiments/perception/worker/build_m48s_rf_detr_tensorrt.sh new file mode 100644 index 0000000..1e8b89d --- /dev/null +++ b/experiments/perception/worker/build_m48s_rf_detr_tensorrt.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -ne 3 ]]; then + echo "usage: $0 ONNX_PATH ENGINE_PATH LOG_PATH" >&2 + exit 2 +fi + +readonly onnx_path="$1" +readonly engine_path="$2" +readonly log_path="$3" +readonly timing_cache="${engine_path}.timing-cache" + +test -f "${onnx_path}" +test ! -e "${engine_path}" +test ! -e "${log_path}" +mkdir -p "$(dirname "${engine_path}")" "$(dirname "${log_path}")" + +/usr/bin/trtexec \ + --onnx="${onnx_path}" \ + --saveEngine="${engine_path}" \ + --timingCacheFile="${timing_cache}" \ + --memPoolSize=workspace:4096 \ + --warmUp=1000 \ + --duration=5 \ + --avgRuns=100 \ + 2>&1 | tee "${log_path}" + +test -s "${engine_path}" +sha256sum "${onnx_path}" "${engine_path}" "${log_path}" diff --git a/experiments/perception/worker/rf_detr_large_config.pbtxt b/experiments/perception/worker/rf_detr_large_config.pbtxt new file mode 100644 index 0000000..7c9f5e2 --- /dev/null +++ b/experiments/perception/worker/rf_detr_large_config.pbtxt @@ -0,0 +1,47 @@ +name: "rf_detr_large" +platform: "tensorrt_plan" +max_batch_size: 0 + +input [ + { + name: "input" + data_type: TYPE_FP32 + dims: [ 1, 3, 704, 704 ] + } +] + +output [ + { + name: "dets" + data_type: TYPE_FP16 + dims: [ 1, 300, 4 ] + }, + { + name: "labels" + data_type: TYPE_FP16 + dims: [ 1, 300, 91 ] + } +] + +instance_group [ + { + count: 1 + kind: KIND_GPU + gpus: [ 0 ] + } +] + +model_warmup [ + { + name: "rf_detr_large_zero" + batch_size: 0 + inputs: { + key: "input" + value: { + data_type: TYPE_FP32 + dims: [ 1, 3, 704, 704 ] + zero_data: true + } + } + } +] diff --git a/src/k1link/perception/detector.py b/src/k1link/perception/detector.py index bf82271..292dfab 100644 --- a/src/k1link/perception/detector.py +++ b/src/k1link/perception/detector.py @@ -1,4 +1,4 @@ -"""Frozen raw-KB4 YOLOX provider for class-agnostic object proposals.""" +"""Versioned fixed-class detector providers for raw-KB4 object proposals.""" from __future__ import annotations @@ -14,10 +14,22 @@ from numpy.typing import NDArray from .contracts import BoundingRegion2D, ObjectProposal2D from .providers import SourcePacket +from .rf_detr_object_detector import ( + RF_DETR_CONFIG, + RF_DETR_MODEL_ID, + RF_DETR_MODEL_VERSION, + RfDetrConfig, + RfDetrDetection, + RfDetrInferenceBackend, + postprocess_rf_detr, + preprocess_raw_kb4_rf_detr, +) from .yolox_object_detector import ( + ALL_COCO_YOLOX_CONFIG, FROZEN_YOLOX_CONFIG, YOLOX_MODEL_ID, YOLOX_MODEL_VERSION, + AllCocoYoloxConfig, FrozenYoloxConfig, ImageResizer, InferenceBackend, @@ -27,8 +39,12 @@ from .yolox_object_detector import ( ) FROZEN_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4/v1" +ALL_COCO_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4-all-coco/v2" FROZEN_YOLOX_MODEL_ID: Final = f"{YOLOX_MODEL_ID}:{YOLOX_MODEL_VERSION}" FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1" +RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0" +RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}" +RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0" class DetectorProviderError(RuntimeError): @@ -57,7 +73,7 @@ class FrozenYoloxDetectorProvider: mask: NDArray[np.bool_], backend: InferenceBackend, resizer: ImageResizer | None = None, - config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG, + config: FrozenYoloxConfig | AllCocoYoloxConfig = FROZEN_YOLOX_CONFIG, clock_ns: Callable[[], int] = time.perf_counter_ns, ) -> None: if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask): @@ -95,7 +111,11 @@ class FrozenYoloxDetectorProvider: ) output = self.backend.infer(tensor) postprocessed = postprocess_yolox(output, self.mask, config=self.config) - proposals = proposals_from_detections(packet, postprocessed.detections) + proposals = proposals_from_detections( + packet, + postprocessed.detections, + provider_id=self.provider_id, + ) except Exception: with self._lock: self._failed_frames += 1 @@ -125,6 +145,8 @@ class FrozenYoloxDetectorProvider: def proposals_from_detections( packet: SourcePacket, detections: tuple[YoloxDetection, ...], + *, + provider_id: str = FROZEN_YOLOX_PROVIDER_ID, ) -> tuple[ObjectProposal2D, ...]: envelope = packet.envelope return tuple( @@ -134,7 +156,7 @@ def proposals_from_detections( frame_id=envelope.frame_id, region=BoundingRegion2D(*detection.bbox_xyxy), objectness=detection.score, - provider_id=FROZEN_YOLOX_PROVIDER_ID, + provider_id=provider_id, model_id=FROZEN_YOLOX_MODEL_ID, preprocess_id=FROZEN_YOLOX_PREPROCESS_ID, semantic_hint=detection.label, @@ -144,12 +166,140 @@ def proposals_from_detections( ) +class AllCocoYoloxDetectorProvider(FrozenYoloxDetectorProvider): + """Emit every qualified COCO class without adding another inference pass.""" + + provider_id: str = ALL_COCO_YOLOX_PROVIDER_ID + + def __init__( + self, + *, + mask: NDArray[np.bool_], + backend: InferenceBackend, + resizer: ImageResizer | None = None, + config: AllCocoYoloxConfig = ALL_COCO_YOLOX_CONFIG, + clock_ns: Callable[[], int] = time.perf_counter_ns, + ) -> None: + super().__init__( + mask=mask, + backend=backend, + resizer=resizer, + config=config, + clock_ns=clock_ns, + ) + + +class RfDetrShadowDetectorProvider: + """Emit behavior-relevant fixed classes from one RF-DETR inference pass.""" + + provider_id: str = RF_DETR_SHADOW_PROVIDER_ID + + def __init__( + self, + *, + mask: NDArray[np.bool_], + backend: RfDetrInferenceBackend, + resizer: ImageResizer | None = None, + config: RfDetrConfig = RF_DETR_CONFIG, + clock_ns: Callable[[], int] = time.perf_counter_ns, + ) -> None: + if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask): + raise DetectorProviderError("RF-DETR valid-FOV mask is incompatible") + self.mask = np.asarray(mask, dtype=np.bool_) + self.backend = backend + self.resizer = resizer + self.config = config + self._clock_ns = clock_ns + self._lock = Lock() + self._input_frames = 0 + self._completed_frames = 0 + self._failed_frames = 0 + self._zero_proposal_frames = 0 + self._proposal_count = 0 + self._rejected: Counter[str] = Counter() + self._core_duration_ns = 0 + + def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]: + payload = packet.image_payload + with self._lock: + self._input_frames += 1 + started_ns = int(self._clock_ns()) + try: + if not isinstance(payload, np.ndarray): + raise DetectorProviderError("RF-DETR requires a decoded BGR image payload") + image = np.asarray(payload) + if image.dtype != np.uint8: + raise DetectorProviderError("decoded BGR image must be uint8") + tensor = preprocess_raw_kb4_rf_detr( + image, + self.mask, + config=self.config, + resizer=self.resizer, + ) + output = self.backend.infer(tensor) + postprocessed = postprocess_rf_detr(output, self.mask, config=self.config) + proposals = proposals_from_rf_detr_detections(packet, postprocessed.detections) + except Exception: + with self._lock: + self._failed_frames += 1 + self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns) + raise + with self._lock: + self._completed_frames += 1 + self._proposal_count += len(proposals) + self._zero_proposal_frames += not proposals + self._rejected.update(dict(postprocessed.rejected)) + self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns) + return proposals + + def snapshot(self) -> DetectorProviderSnapshot: + with self._lock: + return DetectorProviderSnapshot( + input_frames=self._input_frames, + completed_frames=self._completed_frames, + failed_frames=self._failed_frames, + zero_proposal_frames=self._zero_proposal_frames, + proposal_count=self._proposal_count, + rejected=tuple(sorted(self._rejected.items())), + core_duration_ns=self._core_duration_ns, + ) + + +def proposals_from_rf_detr_detections( + packet: SourcePacket, + detections: tuple[RfDetrDetection, ...], +) -> tuple[ObjectProposal2D, ...]: + envelope = packet.envelope + return tuple( + ObjectProposal2D( + proposal_id=f"proposal-{envelope.sequence}-{index}", + source_id=envelope.source_id, + frame_id=envelope.frame_id, + region=BoundingRegion2D(*detection.bbox_xyxy), + objectness=detection.score, + provider_id=RF_DETR_SHADOW_PROVIDER_ID, + model_id=RF_DETR_SHADOW_MODEL_ID, + preprocess_id=RF_DETR_SHADOW_PREPROCESS_ID, + semantic_hint=detection.label, + provider_tracklet=None, + ) + for index, detection in enumerate(detections) + ) + + __all__ = [ + "ALL_COCO_YOLOX_PROVIDER_ID", "FROZEN_YOLOX_MODEL_ID", "FROZEN_YOLOX_PREPROCESS_ID", "FROZEN_YOLOX_PROVIDER_ID", + "RF_DETR_SHADOW_MODEL_ID", + "RF_DETR_SHADOW_PREPROCESS_ID", + "RF_DETR_SHADOW_PROVIDER_ID", "DetectorProviderError", "DetectorProviderSnapshot", + "AllCocoYoloxDetectorProvider", "FrozenYoloxDetectorProvider", + "RfDetrShadowDetectorProvider", "proposals_from_detections", + "proposals_from_rf_detr_detections", ] diff --git a/src/k1link/perception/fixed_class_detector_tournament.py b/src/k1link/perception/fixed_class_detector_tournament.py new file mode 100644 index 0000000..493c731 --- /dev/null +++ b/src/k1link/perception/fixed_class_detector_tournament.py @@ -0,0 +1,285 @@ +"""Contracts for the bounded M48S fixed-class detector tournament.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0" +TOURNAMENT_SCHEMA: Final = "missioncore.m48s-fixed-detector-tournament/v0" +EXACT_FRAME_NAMES: Final = ( + "frame-000121.png", + "frame-000131.png", + "frame-000253.png", + "frame-000275.png", + "frame-000443.png", + "frame-000463.png", + "frame-001094.png", + "frame-001228.png", + "frame-001454.png", + "frame-001856.png", + "frame-002386.png", +) +RISK_GROUPS: Final[Mapping[str, frozenset[str]]] = { + "person": frozenset({"person"}), + "animal": frozenset( + {"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe"} + ), + "light-road-user": frozenset({"bicycle", "motorcycle", "skateboard"}), + "vehicle": frozenset({"car", "bus", "truck"}), +} +_RISK_LABELS: Final = frozenset().union(*RISK_GROUPS.values()) +_FALSE_AUTHORITY: Final = { + "ground_truth": False, + "candidate_accepted": False, + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, +} + + +class FixedClassTournamentError(ValueError): + """Raised when tournament evidence violates its bounded contract.""" + + +@dataclass(frozen=True) +class CandidateDetection: + """One normalized COCO detection emitted by a candidate.""" + + class_id: int + label: str + score: float + bbox_xyxy: tuple[float, float, float, float] + valid_fov_fraction: float + + @classmethod + def from_document(cls, document: object) -> CandidateDetection: + item = _mapping(document, "detection") + box = item.get("bbox_xyxy") + if not isinstance(box, list) or len(box) != 4: + raise FixedClassTournamentError("detection bbox_xyxy must contain four numbers") + values = ( + _number(box[0], "bbox coordinate"), + _number(box[1], "bbox coordinate"), + _number(box[2], "bbox coordinate"), + _number(box[3], "bbox coordinate"), + ) + x1, y1, x2, y2 = values + if x2 <= x1 or y2 <= y1: + raise FixedClassTournamentError("detection box must have positive area") + score = _number(item.get("score"), "detection score") + valid_fov_fraction = _number( + item.get("valid_fov_fraction"), "detection valid-FOV fraction" + ) + if not 0.0 <= score <= 1.0: + raise FixedClassTournamentError("detection score must be in [0, 1]") + if not 0.0 <= valid_fov_fraction <= 1.0: + raise FixedClassTournamentError("valid-FOV fraction must be in [0, 1]") + return cls( + class_id=_integer(item.get("class_id"), "detection class id"), + label=_text(item.get("label"), "detection label"), + score=score, + bbox_xyxy=values, + valid_fov_fraction=valid_fov_fraction, + ) + + @property + def risk_group(self) -> str | None: + for group, labels in RISK_GROUPS.items(): + if self.label in labels: + return group + return None + + +@dataclass(frozen=True) +class CandidateFrame: + """One exact-frame candidate result.""" + + frame_name: str + source_sha256: str + detections: tuple[CandidateDetection, ...] + end_to_end_ms: float + + @classmethod + def from_document(cls, document: object) -> CandidateFrame: + item = _mapping(document, "frame") + detections = item.get("detections") + if not isinstance(detections, list): + raise FixedClassTournamentError("frame detections must be a list") + return cls( + frame_name=_text(item.get("frame_name"), "frame name"), + source_sha256=_digest(item.get("source_sha256"), "source digest"), + detections=tuple(CandidateDetection.from_document(value) for value in detections), + end_to_end_ms=_nonnegative_number( + _mapping(item.get("timing_ms"), "frame timing").get("end_to_end"), + "frame end-to-end timing", + ), + ) + + +@dataclass(frozen=True) +class CandidateWorkerRun: + """Validated raw Worker result for one candidate.""" + + profile_id: str + provider_id: str + upstream_revision: str + checkpoint_sha256: str + frames: tuple[CandidateFrame, ...] + metrics: Mapping[str, object] + authority: Mapping[str, bool] + + @classmethod + def from_document(cls, document: object) -> CandidateWorkerRun: + root = _mapping(document, "worker result") + if root.get("schema_version") != WORKER_RUN_SCHEMA: + raise FixedClassTournamentError("unexpected candidate Worker schema") + frames_raw = root.get("frames") + if not isinstance(frames_raw, list): + raise FixedClassTournamentError("worker result frames must be a list") + frames = tuple(CandidateFrame.from_document(value) for value in frames_raw) + if tuple(sorted(frame.frame_name for frame in frames)) != EXACT_FRAME_NAMES: + raise FixedClassTournamentError("worker result does not contain the exact M48S slice") + if len({frame.frame_name for frame in frames}) != len(EXACT_FRAME_NAMES): + raise FixedClassTournamentError("worker result contains duplicate frames") + execution = _mapping(root.get("execution"), "worker execution") + if execution.get("inference_passes_per_evidence_frame") != 1: + raise FixedClassTournamentError( + "candidate must use one inference pass per evidence frame" + ) + authority = _boolean_mapping(root.get("authority"), "worker authority") + if authority != _FALSE_AUTHORITY: + raise FixedClassTournamentError("candidate Worker result must retain false authority") + completed = root.get("completed") + if completed is not True: + raise FixedClassTournamentError("candidate Worker result is incomplete") + return cls( + profile_id=_text(root.get("profile_id"), "profile id"), + provider_id=_text(root.get("provider_id"), "provider id"), + upstream_revision=_text(root.get("upstream_revision"), "upstream revision"), + checkpoint_sha256=_digest(root.get("checkpoint_sha256"), "checkpoint digest"), + frames=frames, + metrics=_mapping(root.get("metrics"), "worker metrics"), + authority=authority, + ) + + @classmethod + def from_path(cls, path: Path) -> CandidateWorkerRun: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise FixedClassTournamentError( + f"cannot read candidate Worker result: {path}" + ) from error + return cls.from_document(document) + + def quality_summary(self, *, threshold: float) -> dict[str, object]: + if not 0.0 <= threshold <= 1.0: + raise FixedClassTournamentError("quality threshold must be in [0, 1]") + selected = tuple( + detection + for frame in self.frames + for detection in frame.detections + if detection.score >= threshold + ) + class_counts = Counter(item.label for item in selected) + risk_counts = Counter(item.risk_group for item in selected if item.risk_group is not None) + dog_frame = next(frame for frame in self.frames if frame.frame_name == "frame-000253.png") + dog_detections = tuple( + detection + for detection in dog_frame.detections + if detection.label == "dog" and detection.score >= threshold + ) + return { + "threshold": threshold, + "detection_count": len(selected), + "class_counts": dict(sorted(class_counts.items())), + "risk_group_counts": dict(sorted(risk_counts.items())), + "risk_detection_count": sum(1 for item in selected if item.label in _RISK_LABELS), + "frame_000253_dog_detected": bool(dog_detections), + "frame_000253_dog_max_score": ( + round(max(item.score for item in dog_detections), 6) if dog_detections else None + ), + } + + +def canonical_json(value: object) -> bytes: + """Return deterministic JSON bytes for immutable evidence identities.""" + + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def sha256_path(path: Path) -> str: + """Hash a file without loading it into memory.""" + + digest = hashlib.sha256() + try: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as error: + raise FixedClassTournamentError(f"cannot hash evidence file: {path}") from error + return digest.hexdigest() + + +def false_authority() -> dict[str, bool]: + """Return a fresh false-authority document.""" + + return dict(_FALSE_AUTHORITY) + + +def _mapping(value: object, name: str) -> Mapping[str, Any]: + if not isinstance(value, dict): + raise FixedClassTournamentError(f"{name} must be an object") + return value + + +def _boolean_mapping(value: object, name: str) -> Mapping[str, bool]: + mapping = _mapping(value, name) + if set(mapping) != set(_FALSE_AUTHORITY) or not all( + isinstance(item, bool) for item in mapping.values() + ): + raise FixedClassTournamentError(f"{name} must contain the exact boolean authority fields") + return mapping + + +def _text(value: object, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise FixedClassTournamentError(f"{name} must be non-empty text") + return value + + +def _digest(value: object, name: str) -> str: + text = _text(value, name) + if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): + raise FixedClassTournamentError(f"{name} must be a lowercase SHA-256 digest") + return text + + +def _integer(value: object, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise FixedClassTournamentError(f"{name} must be an integer") + return value + + +def _number(value: object, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise FixedClassTournamentError(f"{name} must be numeric") + return float(value) + + +def _nonnegative_number(value: object, name: str) -> float: + result = _number(value, name) + if result < 0.0: + raise FixedClassTournamentError(f"{name} must be non-negative") + return result diff --git a/src/k1link/perception/recorded_source.py b/src/k1link/perception/recorded_source.py index 62629ab..a16fde7 100644 --- a/src/k1link/perception/recorded_source.py +++ b/src/k1link/perception/recorded_source.py @@ -188,20 +188,26 @@ class DecodedRecordedSource: def packets(self, stop_event: Event) -> Iterator[SourcePacket]: images = self.decoder.frames(stop_event) + try: + image: NDArray[np.uint8] | None = next(images) + except StopIteration as exc: + if stop_event.is_set(): + return + raise RecordedSourceError("decoded image stream is empty") from exc for packet in self.source.packets(stop_event): - try: - image = next(images) - except StopIteration as exc: + if image is None: raise RecordedSourceError( "decoded image stream ended before source timeline" - ) from exc + ) if image.shape != (600, 800, 3) or image.dtype != np.uint8: raise RecordedSourceError("decoded image raster is incompatible") yield replace(packet, image_payload=image) - if not stop_event.is_set(): try: - next(images) + image = next(images) except StopIteration: + image = None + if not stop_event.is_set(): + if image is None: return raise RecordedSourceError("decoded image stream exceeds source timeline") diff --git a/src/k1link/perception/rf_detr_object_detector.py b/src/k1link/perception/rf_detr_object_detector.py new file mode 100644 index 0000000..bfbb505 --- /dev/null +++ b/src/k1link/perception/rf_detr_object_detector.py @@ -0,0 +1,401 @@ +"""Pinned RF-DETR-L TensorRT shadow detector for behavior-relevant COCO classes.""" + +from __future__ import annotations + +import http.client +import json +import math +import urllib.parse +from collections import Counter +from dataclasses import dataclass +from typing import Final, Protocol, cast + +import numpy as np +from numpy.typing import NDArray + +from .yolox_object_detector import COCO_CLASSES, ImageResizer, OpenCvBilinearResizer + +RF_DETR_MODEL_ID: Final = "rf_detr_large" +RF_DETR_MODEL_VERSION: Final = 1 +RF_DETR_CHECKPOINT_SHA256: Final = ( + "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38" +) +RF_DETR_ONNX_SHA256: Final = ( + "9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300" +) +RF_DETR_FP16_ONNX_SHA256: Final = ( + "9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb" +) +RF_DETR_ENGINE_SHA256: Final = ( + "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8" +) +COCO_SPARSE_IDS: Final = ( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, + 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 84, 85, 86, 87, 88, 89, 90, +) +COCO_SPARSE_TO_CONTIGUOUS: Final = { + sparse_id: contiguous_id for contiguous_id, sparse_id in enumerate(COCO_SPARSE_IDS) +} +RISK_CLASS_IDS: Final = ( + 0, # person + 1, # bicycle + 2, # car + 3, # motorcycle + 5, # bus + 7, # truck + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, # animals + 36, # skateboard / light road user proxy +) +_MEANS: Final = np.asarray((0.485, 0.456, 0.406), dtype=np.float32) +_STDS: Final = np.asarray((0.229, 0.224, 0.225), dtype=np.float32) + + +class RfDetrDetectorError(RuntimeError): + """The RF-DETR profile, tensor or inference response is incompatible.""" + + +@dataclass(frozen=True, slots=True) +class RfDetrConfig: + source_width: int = 800 + source_height: int = 600 + input_width: int = 704 + input_height: int = 704 + fill_value: int = 114 + minimum_score: float = 0.25 + target_class_ids: tuple[int, ...] = RISK_CLASS_IDS + maximum_detections: int = 300 + minimum_box_area_pixels: float = 64.0 + maximum_box_area_fraction: float = 0.5 + minimum_valid_fov_fraction: float = 0.5 + require_center_inside_valid_fov: bool = True + + def __post_init__(self) -> None: + if ( + self.source_width, + self.source_height, + self.input_width, + self.input_height, + self.fill_value, + self.minimum_score, + self.target_class_ids, + self.maximum_detections, + self.minimum_box_area_pixels, + self.maximum_box_area_fraction, + self.minimum_valid_fov_fraction, + self.require_center_inside_valid_fov, + ) != ( + 800, + 600, + 704, + 704, + 114, + 0.25, + RISK_CLASS_IDS, + 300, + 64.0, + 0.5, + 0.5, + True, + ): + raise RfDetrDetectorError("RF-DETR shadow profile cannot be tuned in place") + + +RF_DETR_CONFIG: Final = RfDetrConfig() + + +@dataclass(frozen=True, slots=True) +class RfDetrRawOutput: + boxes: NDArray[np.float16] + logits: NDArray[np.float16] + + +class RfDetrInferenceBackend(Protocol): + def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput: ... + + +@dataclass(frozen=True, slots=True) +class RfDetrDetection: + class_id: int + label: str + score: float + bbox_xyxy: tuple[float, float, float, float] + valid_fov_fraction: float + + def __post_init__(self) -> None: + if not 0 <= self.class_id < len(COCO_CLASSES): + raise RfDetrDetectorError("RF-DETR class id is invalid") + if self.label != COCO_CLASSES[self.class_id]: + raise RfDetrDetectorError("RF-DETR class label is invalid") + if not math.isfinite(self.score) or not 0.0 <= self.score <= 1.0: + raise RfDetrDetectorError("RF-DETR score is invalid") + x1, y1, x2, y2 = self.bbox_xyxy + if not all(math.isfinite(value) for value in self.bbox_xyxy) or not ( + 0.0 <= x1 < x2 <= 800.0 and 0.0 <= y1 < y2 <= 600.0 + ): + raise RfDetrDetectorError("RF-DETR source bounding box is invalid") + if not 0.0 <= self.valid_fov_fraction <= 1.0: + raise RfDetrDetectorError("RF-DETR valid-FOV fraction is invalid") + + +@dataclass(frozen=True, slots=True) +class RfDetrPostprocessResult: + detections: tuple[RfDetrDetection, ...] + rejected: tuple[tuple[str, int], ...] + + +class TritonRfDetrHttpInferenceBackend: + """Persistent Triton V2 HTTP transport for the strongly typed FP16 engine.""" + + def __init__(self, endpoint: str, *, timeout_seconds: float = 60.0) -> None: + parsed = urllib.parse.urlsplit(endpoint) + if ( + parsed.scheme != "http" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise RfDetrDetectorError("Triton endpoint must be an explicit HTTP origin") + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise RfDetrDetectorError("Triton timeout must be positive") + self.path = ( + f"{parsed.path.rstrip('/')}/v2/models/{RF_DETR_MODEL_ID}" + f"/versions/{RF_DETR_MODEL_VERSION}/infer" + ) + self.connection = http.client.HTTPConnection( + parsed.hostname, + parsed.port or 80, + timeout=timeout_seconds, + ) + + def close(self) -> None: + self.connection.close() + + def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput: + contiguous = np.ascontiguousarray(tensor, dtype=np.float32) + if contiguous.shape != (1, 3, 704, 704) or not np.isfinite(contiguous).all(): + raise RfDetrDetectorError("Triton RF-DETR input tensor is incompatible") + binary = contiguous.tobytes() + header = { + "inputs": [ + { + "name": "input", + "shape": [1, 3, 704, 704], + "datatype": "FP32", + "parameters": {"binary_data_size": len(binary)}, + } + ], + "outputs": [ + {"name": "dets", "parameters": {"binary_data": True}}, + {"name": "labels", "parameters": {"binary_data": True}}, + ], + } + encoded = json.dumps(header, sort_keys=True, separators=(",", ":")).encode() + self.connection.request( + "POST", + self.path, + body=encoded + binary, + headers={ + "Content-Type": "application/octet-stream", + "Inference-Header-Content-Length": str(len(encoded)), + }, + ) + response = self.connection.getresponse() + payload = response.read() + if response.status != 200: + raise RfDetrDetectorError( + f"Triton RF-DETR inference failed with HTTP {response.status}" + ) + header_value = response.getheader("Inference-Header-Content-Length") + try: + header_length = int(header_value or "") + descriptor = json.loads(payload[:header_length]) + outputs = descriptor["outputs"] + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise RfDetrDetectorError("Triton RF-DETR output descriptor is invalid") from exc + if not isinstance(outputs, list) or len(outputs) != 2: + raise RfDetrDetectorError("Triton RF-DETR output count changed") + offset = header_length + arrays: dict[str, NDArray[np.float16]] = {} + for output, expected_name, expected_shape in zip( + outputs, + ("dets", "labels"), + ((1, 300, 4), (1, 300, 91)), + strict=True, + ): + try: + name = output["name"] + datatype = output["datatype"] + shape = tuple(int(value) for value in output["shape"]) + byte_length = int(output["parameters"]["binary_data_size"]) + except (KeyError, TypeError, ValueError) as exc: + raise RfDetrDetectorError( + "Triton RF-DETR output descriptor is incomplete" + ) from exc + expected_bytes = math.prod(expected_shape) * np.dtype(" len(payload) + ): + raise RfDetrDetectorError("Triton RF-DETR output identity changed") + array = np.frombuffer(payload[offset : offset + byte_length], dtype=" NDArray[np.float32]: + if image_bgr.shape != (config.source_height, config.source_width, 3): + raise RfDetrDetectorError("raw KB4 image raster changed") + if image_bgr.dtype != np.uint8 or mask.shape != image_bgr.shape[:2] or mask.dtype != np.bool_: + raise RfDetrDetectorError("raw KB4 image or valid-FOV mask type changed") + masked_bgr = np.where(mask[..., None], image_bgr, config.fill_value).astype(np.uint8) + rgb = np.ascontiguousarray(masked_bgr[:, :, ::-1]) + resized = (resizer or OpenCvBilinearResizer()).resize( + rgb, + config.input_width, + config.input_height, + ) + if resized.shape != (config.input_height, config.input_width, 3): + raise RfDetrDetectorError("resize backend returned an incompatible raster") + normalized = resized.astype(np.float32) / 255.0 + normalized = (normalized - _MEANS) / _STDS + return np.ascontiguousarray(normalized.transpose(2, 0, 1), dtype=np.float32)[None] + + +def postprocess_rf_detr( + output: RfDetrRawOutput, + mask: NDArray[np.bool_], + *, + config: RfDetrConfig = RF_DETR_CONFIG, +) -> RfDetrPostprocessResult: + if output.boxes.shape != (1, 300, 4) or output.logits.shape != (1, 300, 91): + raise RfDetrDetectorError("RF-DETR output tensor shapes are incompatible") + if output.boxes.dtype != np.float16 or output.logits.dtype != np.float16: + raise RfDetrDetectorError("RF-DETR output tensor types are incompatible") + if not np.isfinite(output.boxes).all() or not np.isfinite(output.logits).all(): + raise RfDetrDetectorError("RF-DETR output contains non-finite values") + if mask.shape != (config.source_height, config.source_width) or mask.dtype != np.bool_: + raise RfDetrDetectorError("valid-FOV mask is incompatible") + logits = output.logits[0].astype(np.float32) + probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits, -80.0, 80.0))) + flattened = probabilities.reshape(-1) + topk = np.argsort(-flattened, kind="stable")[: config.maximum_detections] + integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1) + rejected: Counter[str] = Counter() + result: list[RfDetrDetection] = [] + for flat_index in topk: + score = float(flattened[flat_index]) + if score <= config.minimum_score: + continue + query_index = int(flat_index // output.logits.shape[2]) + sparse_class_id = int(flat_index % output.logits.shape[2]) + class_id = COCO_SPARSE_TO_CONTIGUOUS.get(sparse_class_id) + if class_id is None: + rejected["unmapped-class-slot"] += 1 + continue + if class_id not in config.target_class_ids: + rejected["non-risk-class"] += 1 + continue + center_x, center_y, box_width, box_height = ( + float(value) for value in output.boxes[0, query_index].astype(np.float32) + ) + box = np.asarray( + ( + (center_x - box_width / 2.0) * config.source_width, + (center_y - box_height / 2.0) * config.source_height, + (center_x + box_width / 2.0) * config.source_width, + (center_y + box_height / 2.0) * config.source_height, + ), + dtype=np.float32, + ) + box[[0, 2]] = np.clip(box[[0, 2]], 0, config.source_width) + box[[1, 3]] = np.clip(box[[1, 3]], 0, config.source_height) + fraction, center_inside, area = _valid_fraction(box, integral) + if area < config.minimum_box_area_pixels: + rejected["small-box"] += 1 + continue + if area / (config.source_width * config.source_height) > config.maximum_box_area_fraction: + rejected["large-box"] += 1 + continue + if fraction < config.minimum_valid_fov_fraction: + rejected["outside-valid-fov"] += 1 + continue + if config.require_center_inside_valid_fov and not center_inside: + rejected["center-outside-valid-fov"] += 1 + continue + result.append( + RfDetrDetection( + class_id=class_id, + label=COCO_CLASSES[class_id], + score=round(score, 9), + bbox_xyxy=cast( + tuple[float, float, float, float], + tuple(round(float(value), 6) for value in box), + ), + valid_fov_fraction=round(fraction, 6), + ) + ) + result.sort(key=lambda item: (-item.score, item.class_id)) + return RfDetrPostprocessResult(tuple(result), tuple(sorted(rejected.items()))) + + +def _valid_fraction( + box: NDArray[np.float32], integral: NDArray[np.int64] +) -> tuple[float, bool, float]: + height = integral.shape[0] - 1 + width = integral.shape[1] - 1 + x1 = int(np.clip(math.floor(float(box[0])), 0, width)) + y1 = int(np.clip(math.floor(float(box[1])), 0, height)) + x2 = int(np.clip(math.ceil(float(box[2])), 0, width)) + y2 = int(np.clip(math.ceil(float(box[3])), 0, height)) + area = float(max(0, x2 - x1) * max(0, y2 - y1)) + if area <= 0: + return 0.0, False, 0.0 + inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1] + center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1)) + center_y = int(np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1)) + center_inside = bool( + integral[center_y + 1, center_x + 1] + - integral[center_y, center_x + 1] + - integral[center_y + 1, center_x] + + integral[center_y, center_x] + ) + return float(inside) / area, center_inside, area + + +__all__ = [ + "COCO_SPARSE_IDS", + "RF_DETR_CHECKPOINT_SHA256", + "RF_DETR_CONFIG", + "RF_DETR_ENGINE_SHA256", + "RF_DETR_FP16_ONNX_SHA256", + "RF_DETR_MODEL_ID", + "RF_DETR_MODEL_VERSION", + "RF_DETR_ONNX_SHA256", + "RISK_CLASS_IDS", + "RfDetrConfig", + "RfDetrDetection", + "RfDetrDetectorError", + "RfDetrInferenceBackend", + "RfDetrPostprocessResult", + "RfDetrRawOutput", + "TritonRfDetrHttpInferenceBackend", + "postprocess_rf_detr", + "preprocess_raw_kb4_rf_detr", +] diff --git a/src/k1link/perception/yolox_object_detector.py b/src/k1link/perception/yolox_object_detector.py index 4dbc5b8..f404898 100644 --- a/src/k1link/perception/yolox_object_detector.py +++ b/src/k1link/perception/yolox_object_detector.py @@ -43,6 +43,7 @@ COCO_CLASSES: Final = ( "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", ) +ALL_COCO_CLASS_IDS: Final = tuple(range(len(COCO_CLASSES))) class YoloxDetectorError(RuntimeError): @@ -93,6 +94,58 @@ class FrozenYoloxConfig: FROZEN_YOLOX_CONFIG: Final = FrozenYoloxConfig() +@dataclass(frozen=True, slots=True) +class AllCocoYoloxConfig: + """Versioned all-COCO shadow profile using the exact frozen YOLOX tensor.""" + + source_width: int = 800 + source_height: int = 600 + input_width: int = 640 + input_height: int = 640 + fill_value: int = 114 + minimum_score: float = 0.5 + nms_iou_threshold: float = 0.45 + target_class_ids: tuple[int, ...] = ALL_COCO_CLASS_IDS + minimum_box_area_pixels: float = 64.0 + maximum_box_area_fraction: float = 0.5 + minimum_valid_fov_fraction: float = 0.5 + require_center_inside_valid_fov: bool = True + + def __post_init__(self) -> None: + if ( + self.source_width, + self.source_height, + self.input_width, + self.input_height, + self.fill_value, + self.minimum_score, + self.nms_iou_threshold, + self.target_class_ids, + self.minimum_box_area_pixels, + self.maximum_box_area_fraction, + self.minimum_valid_fov_fraction, + self.require_center_inside_valid_fov, + ) != ( + 800, + 600, + 640, + 640, + 114, + 0.5, + 0.45, + ALL_COCO_CLASS_IDS, + 64.0, + 0.5, + 0.5, + True, + ): + raise YoloxDetectorError("all-COCO YOLOX profile cannot be tuned in place") + + +ALL_COCO_YOLOX_CONFIG: Final = AllCocoYoloxConfig() +type YoloxPostprocessConfig = FrozenYoloxConfig | AllCocoYoloxConfig + + @dataclass(frozen=True, slots=True) class YoloxDetection: class_id: int @@ -231,7 +284,7 @@ def preprocess_raw_kb4( image_bgr: NDArray[np.uint8], mask: NDArray[np.bool_], *, - config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG, + config: YoloxPostprocessConfig = FROZEN_YOLOX_CONFIG, resizer: ImageResizer | None = None, ) -> NDArray[np.float32]: if image_bgr.shape != (config.source_height, config.source_width, 3): @@ -261,7 +314,7 @@ def postprocess_yolox( output: NDArray[np.float32], mask: NDArray[np.bool_], *, - config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG, + config: YoloxPostprocessConfig = FROZEN_YOLOX_CONFIG, ) -> YoloxPostprocessResult: if output.shape != (1, 8400, 85) or not np.isfinite(output).all(): raise YoloxDetectorError("YOLOX output tensor is incompatible") @@ -420,9 +473,10 @@ def _sha256(path: Path) -> str: __all__ = [ + "ALL_COCO_CLASS_IDS", "ALL_COCO_YOLOX_CONFIG", "COCO_CLASSES", "YOLOX_CONFIG_SHA256", "YOLOX_MODEL_ID", "YOLOX_MODEL_SHA256", "YOLOX_MODEL_VERSION", "YOLOX_VALID_FOV_SHA256", "FROZEN_YOLOX_CONFIG", - "FrozenYoloxConfig", + "AllCocoYoloxConfig", "FrozenYoloxConfig", "YoloxPostprocessConfig", "ImageResizer", "InferenceBackend", "OpenCvBilinearResizer", "TritonHttpInferenceBackend", "YoloxDetection", "YoloxDetectorError", "YoloxPostprocessResult", "load_valid_fov_mask", "postprocess_yolox", diff --git a/tests/test_fixed_class_detector_tournament.py b/tests/test_fixed_class_detector_tournament.py new file mode 100644 index 0000000..2eb8050 --- /dev/null +++ b/tests/test_fixed_class_detector_tournament.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from k1link.perception.fixed_class_detector_tournament import ( + EXACT_FRAME_NAMES, + WORKER_RUN_SCHEMA, + CandidateWorkerRun, + FixedClassTournamentError, + false_authority, +) + + +def _worker_document() -> dict[str, object]: + frames = [] + for name in EXACT_FRAME_NAMES: + detections = [] + if name == "frame-000253.png": + detections = [ + { + "class_id": 16, + "label": "dog", + "score": 0.72, + "bbox_xyxy": [100.0, 200.0, 160.0, 280.0], + "valid_fov_fraction": 1.0, + } + ] + frames.append( + { + "frame_name": name, + "source_sha256": "a" * 64, + "detections": detections, + "timing_ms": {"end_to_end": 12.0}, + } + ) + return { + "schema_version": WORKER_RUN_SCHEMA, + "profile_id": "candidate/v0", + "provider_id": "shadow-candidate/v0", + "upstream_revision": "revision", + "checkpoint_sha256": "b" * 64, + "completed": True, + "execution": {"inference_passes_per_evidence_frame": 1}, + "frames": frames, + "metrics": {"capacity_fps": 80.0}, + "authority": false_authority(), + } + + +def test_candidate_worker_run_reports_risk_only_quality() -> None: + result = CandidateWorkerRun.from_document(_worker_document()) + + summary = result.quality_summary(threshold=0.5) + + assert summary["detection_count"] == 1 + assert summary["risk_group_counts"] == {"animal": 1} + assert summary["frame_000253_dog_detected"] is True + assert summary["frame_000253_dog_max_score"] == 0.72 + + +def test_candidate_worker_run_rejects_missing_frame() -> None: + document = _worker_document() + frames = document["frames"] + assert isinstance(frames, list) + frames.pop() + + with pytest.raises(FixedClassTournamentError, match="exact M48S slice"): + CandidateWorkerRun.from_document(document) + + +def test_candidate_worker_run_rejects_authority() -> None: + document = deepcopy(_worker_document()) + authority = document["authority"] + assert isinstance(authority, dict) + authority["candidate_accepted"] = True + + with pytest.raises(FixedClassTournamentError, match="false authority"): + CandidateWorkerRun.from_document(document) diff --git a/tests/test_m48s_rf_detr_deployment_result.py b/tests/test_m48s_rf_detr_deployment_result.py new file mode 100644 index 0000000..cd03af9 --- /dev/null +++ b/tests/test_m48s_rf_detr_deployment_result.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import json +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RESULT_ID = ( + "m48s-rf-detr-deployment-gate-" + "2feb9e1b12a5588951ad35d63bf23cf6bdd579d54b5329d46d7696f88c444547" +) +RESULT_ROOT = ( + REPOSITORY_ROOT + / ".runtime/compute-experiments/m48s-semantic-shadow/rf-detr-deployment-results" + / RESULT_ID +) + + +def test_rf_detr_tensorrt_detector_is_ready_only_for_reference_graph_shadow() -> None: + manifest = json.loads((RESULT_ROOT / "manifest.json").read_text("utf-8")) + decision = manifest["decision"] + evidence = manifest["evidence"] + load = evidence["source_paced_load"] + + assert manifest["result_id"] == RESULT_ID + assert manifest["completed"] is True + assert manifest["accepted"] is False + assert decision == { + "detector_source_paced_load_gate_passed": True, + "integrated_world_state_gate_evaluated": False, + "next_gate": ( + "run the RF-DETR shadow provider inside the complete reference graph and " + "require world-state p95 <= 175 ms without changing false authority" + ), + "production_accepted": False, + "ready_for_reference_graph_shadow": True, + "tensorrt_numeric_parity_passed": True, + "tournament_finalist": True, + } + assert evidence["engine_sha256"] == ( + "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8" + ) + assert evidence["tensorrt_parity"]["passed"] is True + assert evidence["tensorrt_parity"]["frame_000253_dog_present_in_tensorrt"] is True + assert evidence["pytorch_quality_at_0_5"]["class_counts"] == evidence[ + "triton_quality_at_0_5" + ]["class_counts"] + assert load["execution"]["source_frames_consumed"] == 18_008 + assert load["execution"]["source_frame_replacements"] == 0 + assert load["execution"]["effective_consumed_fps"] >= 9.5 + assert load["detector_completion_age_ms"]["p95"] <= 175.0 + assert load["gpu"]["gpu_memory_used_mib"]["maximum"] <= 20 * 1024 + assert load["gpu"]["longest_100_percent_gpu_sample_run"] == 0 + assert all(load["checks"].values()) + assert manifest["authority"] == { + "actuation_allowed": False, + "candidate_accepted": False, + "commands_enabled": False, + "ground_truth": False, + "navigation_or_safety_accepted": False, + } diff --git a/tests/test_m48s_yolox_all_coco_result.py b/tests/test_m48s_yolox_all_coco_result.py new file mode 100644 index 0000000..5590700 --- /dev/null +++ b/tests/test_m48s_yolox_all_coco_result.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RESULT_ID = ( + "m48s-yolox-all-coco-shadow-" + "7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06" +) +RESULT_ROOT = ( + REPOSITORY_ROOT + / ".runtime/compute-experiments/m48s-semantic-shadow/yolox-all-coco-results" + / RESULT_ID +) + + +def test_all_coco_yolox_uses_one_inference_pass_with_bounded_postprocess_cost() -> None: + manifest = json.loads((RESULT_ROOT / "manifest.json").read_text("utf-8")) + metrics = manifest["metrics"] + + assert manifest["result_id"] == RESULT_ID + assert manifest["completed"] is True + assert manifest["accepted"] is False + assert metrics["frames"] == {"completed": 11, "requested": 11} + assert metrics["inference_passes_per_frame"] == 1 + assert metrics["frozen_detection_count"] == 44 + assert metrics["all_coco_detection_count"] == 45 + assert metrics["added_detection_count"] == 1 + assert metrics["all_coco_class_counts"] == { + "car": 36, + "handbag": 1, + "person": 3, + "truck": 5, + } + benchmark = metrics["postprocess_benchmark"] + assert benchmark["iterations_per_profile_per_frame"] == 20 + frozen_mean = benchmark["timing_ms"]["frozen_ms"]["mean"] + all_coco_mean = benchmark["timing_ms"]["all_coco_ms"]["mean"] + assert all_coco_mean - frozen_mean < 1.0 + assert metrics["all_coco_core_capacity_fps"] > 30.0 + assert manifest["authority"]["commands_enabled"] is False + assert manifest["authority"]["navigation_or_safety_accepted"] is False diff --git a/tests/test_rf_detr_detector_provider.py b/tests/test_rf_detr_detector_provider.py new file mode 100644 index 0000000..81e9ccd --- /dev/null +++ b/tests/test_rf_detr_detector_provider.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import cast + +import numpy as np +import pytest +from numpy.typing import NDArray + +from k1link.perception.contracts import ( + ClockBasis, + ModalityOutcome, + ModalityStatus, + SourceEnvelope, + TimestampBundle, +) +from k1link.perception.detector import ( + RF_DETR_SHADOW_MODEL_ID, + RF_DETR_SHADOW_PREPROCESS_ID, + RF_DETR_SHADOW_PROVIDER_ID, + RfDetrShadowDetectorProvider, +) +from k1link.perception.providers import SourcePacket +from k1link.perception.rf_detr_object_detector import ( + RF_DETR_CONFIG, + RF_DETR_ENGINE_SHA256, + RF_DETR_FP16_ONNX_SHA256, + RfDetrConfig, + RfDetrDetectorError, + RfDetrRawOutput, + TritonRfDetrHttpInferenceBackend, + postprocess_rf_detr, + preprocess_raw_kb4_rf_detr, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _status() -> ModalityStatus: + return ModalityStatus(True, ModalityOutcome.AVAILABLE, "test-available") + + +def _packet(sequence: int, image: object) -> SourcePacket: + return SourcePacket( + envelope=SourceEnvelope( + source_id="RAVNOVES00", + session_id="20260720T065719Z_viewer_live", + frame_id=f"frame-{sequence:06d}", + sequence=sequence, + timestamps=TimestampBundle( + utc_ns=1_000 + sequence, + monotonic_ns=2_000 + sequence, + source_ns=3_000 + sequence, + clock_basis=ClockBasis.RECORDED_HOST, + ), + source_age_ns=0, + binding_reason="test-recorded-source", + calibration_id="camera-1-kb4-test", + representation_id="registered-map-increment-v1", + image=_status(), + registered_point_increment=_status(), + pose=_status(), + ), + image_payload=image, + registered_point_increment_payload=("points", sequence), + pose_payload=("pose", sequence), + ) + + +class _Resizer: + def __init__(self) -> None: + self.source: NDArray[np.uint8] | None = None + + def resize( + self, image: NDArray[np.uint8], width: int, height: int + ) -> NDArray[np.uint8]: + self.source = image.copy() + output = np.empty((height, width, 3), dtype=np.uint8) + output[:, :, 0] = 255 + output[:, :, 1] = 0 + output[:, :, 2] = 127 + return output + + +class _Backend: + def __init__(self, output: RfDetrRawOutput) -> None: + self.output = output + self.calls = 0 + + def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput: + assert tensor.shape == (1, 3, 704, 704) + assert tensor.dtype == np.float32 + self.calls += 1 + return self.output + + +def _output() -> RfDetrRawOutput: + boxes = np.zeros((1, 300, 4), dtype=np.float16) + logits = np.full((1, 300, 91), -20.0, dtype=np.float16) + boxes[0, 0] = (0.5, 0.5, 0.25, np.float16(1 / 3)) + logits[0, 0, 18] = np.float16(math.log(3.0)) # dog, score 0.75 + boxes[0, 1] = (0.25, 0.25, 0.1, 0.2) + logits[0, 1, 1] = np.float16(math.log(4.0)) # person, score 0.80 + boxes[0, 2] = (0.75, 0.25, 0.1, 0.2) + logits[0, 2, 62] = np.float16(math.log(9.0)) # chair, non-risk + logits[0, 3, 12] = np.float16(math.log(9.0)) # unused COCO slot + return RfDetrRawOutput(boxes=boxes, logits=logits) + + +def test_preprocess_masks_bgr_converts_rgb_stretches_and_normalizes() -> None: + image = np.zeros((600, 800, 3), dtype=np.uint8) + image[:, :] = (10, 20, 30) + mask = np.ones((600, 800), dtype=np.bool_) + mask[0, 0] = False + resizer = _Resizer() + + tensor = preprocess_raw_kb4_rf_detr(image, mask, resizer=resizer) + + assert resizer.source is not None + assert tuple(resizer.source[1, 1]) == (30, 20, 10) + assert tuple(resizer.source[0, 0]) == (114, 114, 114) + assert tensor.shape == (1, 3, 704, 704) + assert tensor.dtype == np.float32 + assert tensor[0, 0, 0, 0] == pytest.approx((1.0 - 0.485) / 0.229) + assert tensor[0, 1, 0, 0] == pytest.approx((0.0 - 0.456) / 0.224) + assert tensor[0, 2, 0, 0] == pytest.approx((127 / 255.0 - 0.406) / 0.225) + + +def test_postprocess_maps_sparse_coco_slots_and_emits_only_risk_classes() -> None: + result = postprocess_rf_detr(_output(), np.ones((600, 800), dtype=np.bool_)) + + assert tuple(item.label for item in result.detections) == ("person", "dog") + assert result.detections[0].score == pytest.approx(0.8, abs=0.001) + assert result.detections[1].score == pytest.approx(0.75, abs=0.001) + assert result.detections[1].bbox_xyxy == pytest.approx( + (300.0, 200.0, 500.0, 400.0), abs=0.03 + ) + assert dict(result.rejected) == {"non-risk-class": 1, "unmapped-class-slot": 1} + + with pytest.raises(RfDetrDetectorError, match="tensor types"): + postprocess_rf_detr( + RfDetrRawOutput( + boxes=cast(NDArray[np.float16], _output().boxes.astype(np.float32)), + logits=_output().logits, + ), + np.ones((600, 800), dtype=np.bool_), + ) + + +def test_shadow_provider_uses_one_pass_and_preserves_semantic_hints() -> None: + backend = _Backend(_output()) + provider = RfDetrShadowDetectorProvider( + mask=np.ones((600, 800), dtype=np.bool_), + backend=backend, + resizer=_Resizer(), + clock_ns=iter((10, 30)).__next__, + ) + + proposals = provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8))) + + assert backend.calls == 1 + assert tuple(item.semantic_hint for item in proposals) == ("person", "dog") + assert all(item.provider_id == RF_DETR_SHADOW_PROVIDER_ID for item in proposals) + assert all(item.model_id == RF_DETR_SHADOW_MODEL_ID for item in proposals) + assert all(item.preprocess_id == RF_DETR_SHADOW_PREPROCESS_ID for item in proposals) + assert provider.snapshot().completed_frames == 1 + assert provider.snapshot().proposal_count == 2 + assert provider.snapshot().core_duration_ns == 20 + + +def test_shadow_profile_is_fixed_and_transport_pins_model_version() -> None: + assert RF_DETR_CONFIG.minimum_score == 0.25 + with pytest.raises(RfDetrDetectorError, match="cannot be tuned"): + RfDetrConfig(minimum_score=0.5) + + backend = TritonRfDetrHttpInferenceBackend("http://127.0.0.1:8100") + try: + assert backend.path == "/v2/models/rf_detr_large/versions/1/infer" + finally: + backend.close() + with pytest.raises(RfDetrDetectorError, match="explicit HTTP origin"): + TritonRfDetrHttpInferenceBackend("http://user:secret@127.0.0.1:8100") + + +def test_shadow_profile_pins_worker_engine_and_retains_false_authority() -> None: + profile = json.loads( + (REPOSITORY_ROOT / "config/perception/rf-detr-large-risk-shadow-v0.json").read_text( + "utf-8" + ) + ) + + assert profile["model"]["strongly_typed_fp16_onnx_sha256"] == RF_DETR_FP16_ONNX_SHA256 + assert ( + profile["model"]["worker_006_rtx4090_tensorrt_11_engine_sha256"] + == RF_DETR_ENGINE_SHA256 + ) + assert profile["emission"]["single_inference_per_source_frame"] is True + assert profile["emission"]["geometry_owns_static_occupancy"] is True + assert profile["emission"]["unlisted_semantic_classes_emitted"] is False + assert not any(profile["authority"].values()) diff --git a/tests/test_yolox_all_coco_profile.py b/tests/test_yolox_all_coco_profile.py new file mode 100644 index 0000000..2c10533 --- /dev/null +++ b/tests/test_yolox_all_coco_profile.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from k1link.perception.detector import ALL_COCO_YOLOX_PROVIDER_ID +from k1link.perception.yolox_object_detector import ( + ALL_COCO_CLASS_IDS, + ALL_COCO_YOLOX_CONFIG, + COCO_CLASSES, + YOLOX_MODEL_SHA256, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +PROFILE_PATH = REPOSITORY_ROOT / "config/perception/yolox-s-all-coco-shadow-v2.json" + + +def test_all_coco_profile_matches_executable_provider_contract() -> None: + profile = json.loads(PROFILE_PATH.read_text("utf-8")) + + assert profile["schema_version"] == "missioncore.yolox-detector-profile/v2" + assert profile["provider_id"] == ALL_COCO_YOLOX_PROVIDER_ID + assert profile["model"]["model_sha256"] == YOLOX_MODEL_SHA256 + assert profile["model"]["additional_inference_passes"] == 0 + assert tuple(profile["postprocess"]["target_class_ids"]) == ALL_COCO_CLASS_IDS + assert ALL_COCO_YOLOX_CONFIG.target_class_ids == tuple(range(len(COCO_CLASSES))) + assert "dog" in profile["class_policy"]["risk_groups"]["animal"] + assert profile["authority"]["commands_enabled"] is False + assert profile["authority"]["navigation_or_safety_accepted"] is False diff --git a/tests/test_yolox_detector_provider.py b/tests/test_yolox_detector_provider.py index 515df1c..9268141 100644 --- a/tests/test_yolox_detector_provider.py +++ b/tests/test_yolox_detector_provider.py @@ -17,13 +17,19 @@ from k1link.perception.contracts import ( TimestampBundle, ) from k1link.perception.detector import ( + ALL_COCO_YOLOX_PROVIDER_ID, FROZEN_YOLOX_PROVIDER_ID, + AllCocoYoloxDetectorProvider, DetectorProviderError, FrozenYoloxDetectorProvider, proposals_from_detections, ) from k1link.perception.providers import SourcePacket from k1link.perception.yolox_object_detector import ( + ALL_COCO_CLASS_IDS, + ALL_COCO_YOLOX_CONFIG, + COCO_CLASSES, + AllCocoYoloxConfig, FrozenYoloxConfig, TritonHttpInferenceBackend, YoloxDetection, @@ -102,6 +108,14 @@ def _one_person_output() -> NDArray[np.float32]: return output +def _one_dog_output() -> NDArray[np.float32]: + output = np.zeros((1, 8400, 85), dtype=np.float32) + output[0, 0, :4] = [40.0, 30.0, math.log(10.0), math.log(10.0)] + output[0, 0, 4] = 0.9 + output[0, 0, 5 + 16] = 0.9 + return output + + def test_frozen_preprocess_and_postprocess_match_the_e46j_contract() -> None: image = np.full((600, 800, 3), 7, dtype=np.uint8) mask = np.ones((600, 800), dtype=np.bool_) @@ -174,6 +188,39 @@ def test_frozen_profile_rejects_in_place_threshold_tuning() -> None: FrozenYoloxConfig(minimum_score=0.51) +def test_all_coco_profile_emits_dog_without_another_inference_pass() -> None: + mask = np.ones((600, 800), dtype=np.bool_) + + assert postprocess_yolox(_one_dog_output(), mask).detections == () + all_coco = postprocess_yolox( + _one_dog_output(), + mask, + config=ALL_COCO_YOLOX_CONFIG, + ) + backend = _Backend(_one_dog_output()) + provider = AllCocoYoloxDetectorProvider( + mask=mask, + backend=backend, + resizer=_Resizer(), + ) + proposals = provider.detect( + _packet(16, np.zeros((600, 800, 3), dtype=np.uint8)) + ) + + assert tuple(range(80)) == ALL_COCO_CLASS_IDS + assert len(COCO_CLASSES) == 80 + assert tuple(item.label for item in all_coco.detections) == ("dog",) + assert backend.calls == 1 + assert len(proposals) == 1 + assert proposals[0].provider_id == ALL_COCO_YOLOX_PROVIDER_ID + assert proposals[0].semantic_hint == "dog" + + +def test_all_coco_profile_is_versioned_and_cannot_be_tuned_in_place() -> None: + with pytest.raises(YoloxDetectorError, match="all-COCO.*cannot be tuned"): + AllCocoYoloxConfig(target_class_ids=(0, 16)) + + def test_triton_transport_pins_the_frozen_model_version() -> None: backend = TritonHttpInferenceBackend("http://127.0.0.1:8000") try: