SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): evaluate fixed-class detector candidates

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 16:44:15 +03:00
parent 6276bbf324
commit 33cef2fdea
28 changed files with 4498 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/perception/fixed-class-detector-tournament-v0.json
+89
View File
@@ -0,0 +1,89 @@
{
"schema_version": "missioncore.fixed-class-detector-tournament-profile/v0",
"profile_id": "m48s-fixed-class-detector-tournament/v0",
"source": {
"slice_id": "m48s-risk-11-valid-fov-fill/v1",
"frame_count": 11,
"valid_fov_result_id": "valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2",
"baseline_result_id": "m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
},
"candidates": [
{
"profile_id": "dfine-s-coco-640-fp16/v0",
"provider_id": "shadow-dfine-s-coco/v0",
"role": "fast",
"upstream": {
"repository": "https://github.com/Peterande/D-FINE",
"revision": "956d1709314c2c6a4df6f34de232054578a7449f",
"license": "Apache-2.0"
},
"checkpoint": {
"url": "https://github.com/Peterande/storage/releases/download/dfinev1.0/dfine_s_coco.pth",
"training_basis": "COCO-only",
"sha256": "48a6c8cc43eb57186843f752e2e8461ddd3326e0d3c575e71e6e960844683e89"
},
"model": {
"dataset_vocabulary": "COCO-80",
"input_width": 640,
"input_height": 640,
"published_coco_ap_50_95": 48.5,
"published_t4_tensorrt_fp16_ms": 3.49,
"published_parameter_count_millions": 10
},
"deployment_path": ["PyTorch qualification", "ONNX", "TensorRT FP16", "Triton"]
},
{
"profile_id": "rf-detr-large-coco-704-fp16/v0",
"provider_id": "shadow-rf-detr-large-coco/v0",
"role": "strong",
"upstream": {
"repository": "https://github.com/roboflow/rf-detr",
"version": "1.9.4",
"revision": "9b009fa928d6218320439803d1da01869a85c072",
"license": "Apache-2.0"
},
"checkpoint": {
"url": "https://storage.googleapis.com/rfdetr/rf-detr-large-2026.pth",
"training_basis": "Apache-designated COCO checkpoint",
"expected_md5": "5cb72153541cbcb9aa6efa26222acc75",
"sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38"
},
"model": {
"dataset_vocabulary": "COCO-80",
"input_width": 704,
"input_height": 704,
"published_coco_ap_50_95": 56.5,
"published_t4_tensorrt_fp16_ms": 6.8,
"published_parameter_count_millions": 33.9
},
"deployment_path": ["PyTorch qualification", "ONNX", "TensorRT FP16", "Triton"]
}
],
"emission": {
"class_vocabulary": "COCO-80",
"qualification_score_thresholds": [0.25, 0.5],
"geometry_owns_occupancy": true,
"static_object_names_have_behavior_authority": false,
"inference_passes_per_frame": 1
},
"risk_policy": {
"classified_groups": ["person", "animal", "light-road-user", "vehicle"],
"unknown_moving_response": "conservative-risk",
"unknown_stationary_response": "route-around"
},
"promotion_gates": {
"minimum_recorded_source_fps": 9.5,
"maximum_world_state_p95_ms": 175,
"maximum_worker_vram_gib": 20,
"sustained_gpu_utilization_below_percent": 100,
"queue_policy": "bounded-latest-wins",
"full_load_duration_minutes": [30, 60]
},
"authority": {
"ground_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
config/perception/rf-detr-large-risk-shadow-v0.json
+81
View File
@@ -0,0 +1,81 @@
{
"schema_version": "missioncore.rf-detr-risk-shadow-profile/v0",
"profile_id": "rf-detr-large-coco-704-trt11-fp16-risk-shadow/v0",
"provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
"model": {
"model_id": "rf_detr_large",
"model_version": 1,
"upstream_version": "1.9.4",
"upstream_revision": "9b009fa928d6218320439803d1da01869a85c072",
"checkpoint_sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38",
"exported_onnx_sha256": "9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300",
"strongly_typed_fp16_onnx_sha256": "9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb",
"worker_006_rtx4090_tensorrt_11_engine_sha256": "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8",
"input": {
"name": "input",
"datatype": "FP32",
"shape": [1, 3, 704, 704]
},
"outputs": [
{"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]},
{"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]}
]
},
"preprocessing": {
"source_raster": [800, 600],
"source_color": "BGR",
"model_color": "RGB",
"valid_fov_fill_value": 114,
"resize": "704x704-bilinear-antialias-false",
"normalization_mean": [0.485, 0.456, 0.406],
"normalization_std": [0.229, 0.224, 0.225]
},
"emission": {
"single_inference_per_source_frame": true,
"minimum_score": 0.25,
"maximum_topk_query_class_pairs": 300,
"behavior_relevant_classes": [
"person",
"bicycle",
"car",
"motorcycle",
"bus",
"truck",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"skateboard"
],
"geometry_owns_static_occupancy": true,
"unlisted_semantic_classes_emitted": false,
"minimum_box_area_pixels": 64,
"maximum_box_area_fraction": 0.5,
"minimum_valid_fov_fraction": 0.5,
"require_center_inside_valid_fov": true
},
"queue": {
"policy": "bounded-latest-wins",
"capacity": 2
},
"status": {
"tournament_finalist": true,
"tensorrt_parity_passed": true,
"detector_load_gate_passed": true,
"integrated_world_state_gate_passed": false,
"production_accepted": false
},
"authority": {
"ground_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
config/perception/yolox-s-all-coco-shadow-v2.json
+60
View File
@@ -0,0 +1,60 @@
{
"schema_version": "missioncore.yolox-detector-profile/v2",
"profile_id": "yolox-s-raw-kb4-all-coco-shadow/v2",
"provider_id": "triton-yolox-s-raw-kb4-all-coco/v2",
"model": {
"model_id": "yolox_s:1",
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
"dataset_vocabulary": "COCO-80",
"additional_inference_passes": 0
},
"preprocess": {
"source_width": 800,
"source_height": 600,
"input_width": 640,
"input_height": 640,
"fill_value": 114,
"valid_fov_mask_sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
},
"postprocess": {
"minimum_score": 0.5,
"nms_iou_threshold": 0.45,
"target_class_ids": [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
70, 71, 72, 73, 74, 75, 76, 77, 78, 79
],
"minimum_box_area_pixels": 64.0,
"maximum_box_area_fraction": 0.5,
"minimum_valid_fov_fraction": 0.5,
"require_center_inside_valid_fov": true
},
"class_policy": {
"emission": "all-qualified-coco-classes",
"behavior_authority": "separate-risk-policy-only",
"risk_groups": {
"person": ["person"],
"animal": [
"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear",
"zebra", "giraffe"
],
"light-road-user": ["bicycle", "motorcycle", "skateboard"],
"vehicle": ["car", "bus", "truck"]
},
"unmapped_label_response": "advisory-only",
"unknown_moving_response": "conservative-risk",
"unknown_stationary_response": "route-around"
},
"authority": {
"ground_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
experiments/perception/M48S_FIXED_CLASS_DETECTOR_TOURNAMENT_REPORT.md
+133
View File
@@ -0,0 +1,133 @@
# M48S fixed-class detector tournament и Worker deployment gate
Дата: 2026-08-25
Режим: experimental shadow
Worker: `worker-006`, NVIDIA GeForce RTX 4090
Production acceptance: **нет**
## Решение
RF-DETR-L выиграл bounded-турнир, был экспортирован в ONNX, преобразован в
strongly typed FP16 graph, собран TensorRT 11 и проверен через изолированный
Triton. Detector-only нагрузочный gate длиной 30 минут принят. Кандидат готов к
следующему shadow-gate внутри полного reference graph, но не получил
navigation, safety, command или actuation authority.
Production critical path не включает Mask Grounding DINO, SAM 2 или OpenCLIP.
Статические неизвестные объекты остаются ответственностью geometry/occupancy и
объезжаются без расхода detector inference на их название. Один fixed-class
проход камеры используется только для поведенчески значимых классов.
## Поведенческая граница
RF-DETR shadow provider выпускает только:
- `person`;
- `bicycle`, `motorcycle`, `skateboard`;
- `car`, `bus`, `truck`;
- COCO animal classes, включая `dog`.
Урны, столбы, полусферы, бордюры и прочие статические препятствия не обязаны
получать семантическое имя: их наличие и геометрия принадлежат class-free
occupancy. Неизвестный движущийся объект остаётся conservative. Отдельного
COCO-класса `scooter` нет, поэтому самокат пока нельзя считать надёжно
классифицированным: до отдельного admission gate он остаётся geometry/motion
hazard, а не безопасным отрицанием.
## Турнир на immutable 11-frame slice
Все профили выполняли один inference pass на кадр; ручная проверка не объявлена
ground truth.
| Профиль | Core capacity | p95 | Собака на frame 253 | Решение |
|---|---:|---:|---|---|
| YOLOX-S all-COCO/v2 | 39,136 FPS | 38,267 мс | нет | regression baseline |
| D-FINE-S COCO FP16 | 31,100 FPS | 42,945 мс | нет при 0,25 и 0,5 | отклонён |
| RF-DETR-L COCO FP16 | 44,786 FPS | 32,588 мс | да, score 0,740723 | finalist |
D-FINE также давал заметные semantic confusions: собака как `skateboard`, корпус
сканера как `surfboard`, дублирующиеся risk-labels на одном объекте. RF-DETR на
этом slice дал более чистые person/vehicle labels и корректную собаку.
Immutable tournament result:
`m48s-fixed-detector-tournament-0e61d75e6dc575d53e4bb98772a41d240fe627ad642de5178beb1154636e1299`.
## TensorRT/Triton квалификация
Закреплены следующие identities:
- upstream RF-DETR revision: `9b009fa928d6218320439803d1da01869a85c072`;
- checkpoint SHA-256: `0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38`;
- exported ONNX SHA-256: `9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300`;
- strongly typed FP16 ONNX SHA-256: `9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb`;
- Worker 006 TensorRT engine SHA-256: `986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8`.
TensorRT parity на frame 253:
- PyTorch dog score: 0,740723;
- TensorRT dog score: 0,741674;
- абсолютная разница score: 0,000951;
- box IoU: 0,990117;
- class counts при threshold 0,5 совпадают точно: 57 `car`, 8 `truck`,
6 `person`, 1 `dog`, 1 `fire hydrant`.
100-iteration Triton benchmark: 42,496 FPS end-to-end; mean 23,531 мс; p95
34,156 мс. Production Triton во время проверки не изменялся: использовался
отдельный безпортовый Triton-контейнер в namespace эксперимента.
## 30-минутный source-paced gate
Источник: RAVNOVES00, SHA-256
`cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8`,
10,0039 FPS. Durable background services на Worker оставались включёнными.
| Метрика | Результат | Gate |
|---|---:|---:|
| Время | 1800,020 с | ≥ 1800 с |
| Кадры | 18 008 produced / 18 008 consumed | без потерь |
| Effective FPS | 10,004 | ≥ 9,5 |
| Detector end-to-end p95 | 32,415 мс | наблюдение |
| Detector completion age p95 | 40,621 мс | ≤ 175 мс |
| Queue | max depth 1/2, replacements 0 | bounded latest-wins |
| GPU utilization | mean 51,408%, p95 55%, max 65% | без sustained 100% |
| Worker VRAM | mean 9542,7 MiB, max 9556 MiB | ≤ 20 GiB |
| Ошибки | 0 | 0 |
Все восемь автоматических load checks приняты. Это detector-only gate, поэтому
он не доказывает p95 полного world state, качество tracker association или
корректность risk-policy.
Immutable deployment result:
`m48s-rf-detr-deployment-gate-2feb9e1b12a5588951ad35d63bf23cf6bdd579d54b5329d46d7696f88c444547`.
## Реализация
- `src/k1link/perception/rf_detr_object_detector.py` — pinned preprocessing,
Triton V2 binary HTTP backend, FP16 output validation, fixed risk-class
qualification и fail-closed FOV/area gates.
- `src/k1link/perception/detector.py` — `RfDetrShadowDetectorProvider`, один
inference pass и semantic hints без authority.
- `config/perception/rf-detr-large-risk-shadow-v0.json` — неизменяемый профиль,
threshold 0,25, bounded latest-wins queue capacity 2, geometry-owned static
occupancy.
- `experiments/perception/worker/` — воспроизводимые export/build/Triton
declarations.
- `experiments/perception/run_m48s_rf_detr_load_worker.py` — source-paced
concurrent-load gate с GPU, queue и Triton accounting.
- `experiments/perception/seal_m48s_rf_detr_deployment.py` — content-addressed
immutable seal с false authority.
## Следующий gate
Подключить этот provider в полный reference graph вместе с существующими
geometry observations, tracker и advisory risk-policy. На том же записанном
источнике и при сохранённых Worker services требуется:
1. world-state p95 не более 175 мс;
2. не менее 9,5 source FPS, bounded latest-wins без неучтённых потерь;
3. стабильные track identities и conservative unknown-moving handling;
4. раздельные реакции на person/animal/light-road-user/vehicle;
5. отсутствие navigation/safety/command authority до отдельного acceptance.
Только после этого можно решать вопрос о замене текущего production detector.
Текущий результат разрешает reference-graph shadow, а не production switch.
experiments/perception/M48S_YOLOX_ALL_COCO_REPORT.md
+62
View File
@@ -0,0 +1,62 @@
# M48S YOLOX-S all-COCO shadow report
Date: 2026-08-25
Status: executable shadow completed; full-load promotion gate open
## Why six classes were previously emitted
The accepted `triton-yolox-s-raw-kb4/v1` provider was deliberately frozen on
COCO ids `0, 1, 2, 3, 5, 7`: person, bicycle, car, motorcycle, bus and truck.
That was a bounded detector qualification and reproducibility boundary, not an
inference optimization. YOLOX-S already returns an `[1, 8400, 85]` tensor with
all 80 COCO class scores. The six-class filter ran after the single inference.
The old provider, hashes and M4 replay results remain unchanged. The new
`triton-yolox-s-raw-kb4-all-coco/v2` provider uses the same model, tensor,
preprocess, thresholds and valid-FOV gates, but emits every qualified COCO class.
## Worker comparison
Both profiles were applied to the exact same tensor response for each of the 11
M48S frames. The client shared the Worker's `mission-core-compute_default`
network with Triton, avoiding host-NAT tensor transport.
| Measure | Frozen six classes | All COCO-80 |
|---|---:|---:|
| Inference passes per frame | 1 | 1, shared |
| Detections | 44 | 45 |
| Postprocess mean, 220 balanced iterations | 6.430 ms | 6.725 ms |
| Postprocess p50 | 5.671 ms | 5.679 ms |
| Postprocess p95 | 11.013 ms | 11.592 ms |
The measured mean postprocess difference was `0.295 ms`; the p50 difference was
`0.008 ms`. The combined preprocess + inference + all-COCO postprocess capacity
was `39.136 FPS` on this bounded slice. This is a capacity diagnostic, not a
full-route load acceptance.
The only newly emitted detection was `handbag` on frame 253, correctly covering
the bag carried by the visible person. The class counts were 36 car, five truck,
three person and one handbag. The visible dog on frame 253 was not detected.
Removing the filter therefore exposes all model answers at negligible compute
cost, but does not repair classes the model fails to recognize.
Immutable result:
`m48s-yolox-all-coco-shadow-7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06`.
## Policy boundary
All qualified COCO labels are now available to downstream consumers. Emission
does not grant every class behavioral authority:
- person, animal, light road user and vehicle labels may enter a separately
versioned risk policy after qualification;
- other labels remain advisory diagnostics;
- geometry owns occupancy for every object;
- unknown moving objects retain conservative risk;
- unknown stationary objects remain route-around;
- commands, actuation, navigation and safety authority remain false.
The next gate is the full recorded source under representative concurrent Worker
load. It must compare source delivery, detector FPS, p95 latency, queue depth,
drops, GPU utilization and VRAM against the frozen six-class baseline before v2
can replace v1 in the production assembly.
experiments/perception/convert_m48s_rf_detr_onnx_fp16.py
+156
View File
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""Convert the pinned RF-DETR ONNX graph to a strongly typed FP16 graph."""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
from typing import Any, Final
import onnx # type: ignore[import-not-found]
from onnx import TensorProto
from onnxconverter_common import float16 # type: ignore[import-not-found]
SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-onnx-fp16-conversion/v3"
FALSE_AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--expected-input-sha256", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
arguments = parser.parse_args()
source = arguments.input.resolve(strict=True)
source_sha256 = sha256_path(source)
if source_sha256 != arguments.expected_input_sha256:
raise RuntimeError("source ONNX SHA-256 does not match the export manifest")
output = arguments.output.absolute()
manifest = arguments.manifest.absolute()
if output.exists() or manifest.exists():
raise RuntimeError("FP16 ONNX output or manifest already exists")
started_utc_ns = time.time_ns()
graph = onnx.load(str(source))
converted = float16.convert_float_to_float16(
graph,
keep_io_types=False,
disable_shape_infer=False,
)
retargeted_casts = _retarget_float_casts_to_fp16(converted)
_insert_fp32_input_cast(converted)
onnx.checker.check_model(converted)
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
onnx.save(converted, str(output))
verified = onnx.load(str(output), load_external_data=False)
onnx.checker.check_model(verified)
input_types = {value.name: value.type.tensor_type.elem_type for value in verified.graph.input}
output_types = {
value.name: value.type.tensor_type.elem_type for value in verified.graph.output
}
if input_types != {"input": TensorProto.FLOAT}:
raise RuntimeError(f"FP16 ONNX input boundary is not FLOAT: {input_types}")
if output_types != {"dets": TensorProto.FLOAT16, "labels": TensorProto.FLOAT16}:
raise RuntimeError(f"FP16 ONNX outputs are not FLOAT16: {output_types}")
initializer_counts = _initializer_type_counts(verified)
if initializer_counts.get("FLOAT16", 0) == 0:
raise RuntimeError("FP16 ONNX has no FLOAT16 initializers")
document = {
"schema_version": SCHEMA_VERSION,
"profile_id": "rf-detr-large-coco-704-trt11-fp16/v0",
"source_onnx_sha256": source_sha256,
"output_onnx_sha256": sha256_path(output),
"output_size_bytes": output.stat().st_size,
"boundary_types": {"inputs": input_types, "outputs": output_types},
"initializer_type_counts": initializer_counts,
"float_casts_retargeted_to_fp16": retargeted_casts,
"started_utc_ns": started_utc_ns,
"completed_utc_ns": time.time_ns(),
"completed": True,
"authority": FALSE_AUTHORITY,
}
manifest.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
manifest.write_bytes(canonical_json(document) + b"\n")
print(output)
print(json.dumps(document, indent=2, sort_keys=True))
return 0
def _initializer_type_counts(graph: Any) -> dict[str, int]:
counts: dict[str, int] = {}
for initializer in graph.graph.initializer:
name = TensorProto.DataType.Name(initializer.data_type)
counts[name] = counts.get(name, 0) + 1
return dict(sorted(counts.items()))
def _insert_fp32_input_cast(graph: Any) -> None:
"""Keep a conventional FP32 client boundary before the strongly typed FP16 graph."""
input_value = next((item for item in graph.graph.input if item.name == "input"), None)
if input_value is None:
raise RuntimeError("RF-DETR graph has no input tensor named 'input'")
if input_value.type.tensor_type.elem_type != TensorProto.FLOAT16:
raise RuntimeError("RF-DETR converted input is not FLOAT16 before boundary adaptation")
cast_output = "missioncore_input_fp16"
for node in graph.graph.node:
for index, name in enumerate(node.input):
if name == "input":
node.input[index] = cast_output
cast = onnx.helper.make_node(
"Cast",
inputs=["input"],
outputs=[cast_output],
name="missioncore_input_fp32_to_fp16",
to=TensorProto.FLOAT16,
)
graph.graph.node.insert(0, cast)
input_value.type.tensor_type.elem_type = TensorProto.FLOAT
def _retarget_float_casts_to_fp16(graph: Any) -> int:
"""Retarget explicit PyTorch FLOAT casts that would re-expand an FP16 data path."""
count = 0
for node in graph.graph.node:
if node.op_type != "Cast":
continue
for attribute in node.attribute:
if attribute.name == "to" and attribute.i == TensorProto.FLOAT:
attribute.i = TensorProto.FLOAT16
count += 1
if count == 0:
raise RuntimeError("RF-DETR graph has no FLOAT casts to retarget")
return count
def sha256_path(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/export_m48s_rf_detr_worker.py
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Export the pinned M48S RF-DETR finalist to a static ONNX artifact."""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from collections.abc import Iterable
from pathlib import Path
from typing import Any, Final
import onnx # type: ignore[import-not-found]
SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-onnx-export/v0"
FALSE_AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--expected-checkpoint-sha256", required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--upstream-revision", required=True)
arguments = parser.parse_args()
checkpoint = arguments.checkpoint.resolve(strict=True)
checkpoint_sha256 = sha256_path(checkpoint)
if checkpoint_sha256 != arguments.expected_checkpoint_sha256:
raise RuntimeError("checkpoint SHA-256 does not match the pinned finalist")
output_root = arguments.output_root.absolute()
manifest_path = arguments.manifest.absolute()
if output_root.exists():
raise RuntimeError("ONNX output root already exists")
if manifest_path.exists():
raise RuntimeError("ONNX export manifest already exists")
from rfdetr import RFDETRLarge # type: ignore[import-not-found]
started_utc_ns = time.time_ns()
model = RFDETRLarge(pretrain_weights=str(checkpoint))
exported_path = Path(
model.export(
output_dir=str(output_root),
format="onnx",
shape=(704, 704),
batch_size=1,
dynamic_batch=False,
opset_version=17,
verbose=False,
notes={
"missioncore_profile_id": "rf-detr-large-coco-704-fp16/v0",
"upstream_revision": arguments.upstream_revision,
"checkpoint_sha256": checkpoint_sha256,
"authority": FALSE_AUTHORITY,
},
)
).resolve(strict=True)
graph = onnx.load(str(exported_path), load_external_data=False)
onnx.checker.check_model(graph)
inputs = [_tensor_description(value) for value in graph.graph.input]
outputs = [_tensor_description(value) for value in graph.graph.output]
expected_input = [{"name": "input", "element_type": 1, "shape": [1, 3, 704, 704]}]
if inputs != expected_input:
raise RuntimeError(f"unexpected RF-DETR ONNX input contract: {inputs}")
if [item["name"] for item in outputs] != ["dets", "labels"]:
raise RuntimeError(f"unexpected RF-DETR ONNX outputs: {outputs}")
document = {
"schema_version": SCHEMA_VERSION,
"profile_id": "rf-detr-large-coco-704-fp16/v0",
"provider_id": "shadow-rf-detr-large-coco-onnx/v0",
"upstream_revision": arguments.upstream_revision,
"checkpoint_sha256": checkpoint_sha256,
"onnx": {
"path": str(exported_path),
"sha256": sha256_path(exported_path),
"size_bytes": exported_path.stat().st_size,
"opset_imports": [
{"domain": item.domain, "version": item.version}
for item in graph.opset_import
],
"inputs": inputs,
"outputs": outputs,
},
"started_utc_ns": started_utc_ns,
"completed_utc_ns": time.time_ns(),
"completed": True,
"authority": FALSE_AUTHORITY,
}
manifest_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
manifest_path.write_bytes(canonical_json(document) + b"\n")
print(exported_path)
print(json.dumps(document["onnx"], indent=2, sort_keys=True))
return 0
def _tensor_description(value: Any) -> dict[str, object]:
tensor = value.type.tensor_type
return {
"name": value.name,
"element_type": tensor.elem_type,
"shape": [_dimension_value(item) for item in tensor.shape.dim],
}
def _dimension_value(value: Any) -> int | str | None:
if value.HasField("dim_value"):
return int(value.dim_value)
if value.HasField("dim_param"):
return str(value.dim_param)
return None
def sha256_path(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
def _names(values: Iterable[dict[str, object]]) -> tuple[object, ...]:
"""Keep static analyzers honest when ONNX collections are inspected in tests."""
return tuple(value.get("name") for value in values)
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_m48s_all_coco_yolox_worker.py
+288
View File
@@ -0,0 +1,288 @@
#!/usr/bin/env python3
"""Compare frozen road classes with all COCO classes on one YOLOX tensor pass."""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from collections import Counter
from pathlib import Path
from typing import Any, Final
import numpy as np
from PIL import Image
from k1link.perception.yolox_object_detector import (
ALL_COCO_YOLOX_CONFIG,
COCO_CLASSES,
FROZEN_YOLOX_CONFIG,
YOLOX_MODEL_SHA256,
TritonHttpInferenceBackend,
YoloxPostprocessConfig,
load_valid_fov_mask,
postprocess_yolox,
preprocess_raw_kb4,
)
SCHEMA: Final = "missioncore.m48s-yolox-all-coco-shadow/v0"
POSTPROCESS_BENCHMARK_ITERATIONS: Final = 20
AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input-root", type=Path, required=True)
parser.add_argument("--valid-fov-mask", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument(
"--triton-endpoint",
default="http://host.docker.internal:8000",
)
arguments = parser.parse_args()
input_root = arguments.input_root.resolve(strict=True)
images = tuple(sorted(input_root.glob("frame-*.jpg")))
if len(images) != 11:
raise RuntimeError("M48S all-COCO shadow requires the exact 11-frame slice")
output_parent = arguments.output_root.expanduser().absolute()
output_parent.mkdir(mode=0o700, parents=True, exist_ok=True)
mask = load_valid_fov_mask(arguments.valid_fov_mask)
backend = TritonHttpInferenceBackend(arguments.triton_endpoint)
rows = []
totals: Counter[str] = Counter()
timings: dict[str, list[float]] = {
"decode_ms": [],
"preprocess_ms": [],
"inference_ms": [],
"frozen_postprocess_ms": [],
"all_coco_postprocess_ms": [],
"all_coco_core_ms": [],
}
postprocess_benchmark: dict[str, list[float]] = {
"frozen_ms": [],
"all_coco_ms": [],
}
try:
for image_path in images:
started = time.perf_counter_ns()
with Image.open(image_path) as opened:
rgb = np.asarray(opened.convert("RGB"), dtype=np.uint8)
bgr = np.ascontiguousarray(rgb[:, :, ::-1])
decoded = time.perf_counter_ns()
tensor = preprocess_raw_kb4(bgr, mask, config=ALL_COCO_YOLOX_CONFIG)
preprocessed = time.perf_counter_ns()
output = backend.infer(tensor)
inferred = time.perf_counter_ns()
frozen = postprocess_yolox(output, mask, config=FROZEN_YOLOX_CONFIG)
frozen_postprocessed = time.perf_counter_ns()
all_coco = postprocess_yolox(output, mask, config=ALL_COCO_YOLOX_CONFIG)
all_postprocessed = time.perf_counter_ns()
_benchmark_postprocess(
output,
mask,
destination=postprocess_benchmark,
)
frozen_ids = {
_detection_identity(item.class_id, item.score, item.bbox_xyxy)
for item in frozen.detections
}
added = tuple(
item
for item in all_coco.detections
if _detection_identity(item.class_id, item.score, item.bbox_xyxy)
not in frozen_ids
)
totals["frame_count"] += 1
totals["frozen_detection_count"] += len(frozen.detections)
totals["all_coco_detection_count"] += len(all_coco.detections)
totals["added_detection_count"] += len(added)
for item in all_coco.detections:
totals[f"class:{item.label}"] += 1
timings["decode_ms"].append(_milliseconds(started, decoded))
timings["preprocess_ms"].append(_milliseconds(decoded, preprocessed))
timings["inference_ms"].append(_milliseconds(preprocessed, inferred))
timings["frozen_postprocess_ms"].append(
_milliseconds(inferred, frozen_postprocessed)
)
timings["all_coco_postprocess_ms"].append(
_milliseconds(frozen_postprocessed, all_postprocessed)
)
timings["all_coco_core_ms"].append(
_milliseconds(decoded, preprocessed)
+ _milliseconds(preprocessed, inferred)
+ _milliseconds(frozen_postprocessed, all_postprocessed)
)
rows.append(
{
"frame_name": image_path.name,
"source_sha256": _sha256(image_path),
"frozen_detections": [
_detection_document(item) for item in frozen.detections
],
"all_coco_detections": [
_detection_document(item) for item in all_coco.detections
],
"added_detections": [_detection_document(item) for item in added],
"timing_ms": {name: values[-1] for name, values in timings.items()},
"authority": AUTHORITY,
}
)
finally:
backend.close()
frame_bytes = b"".join(_canonical_json(item) + b"\n" for item in rows)
timing_metrics = {name: _timing_summary(values) for name, values in timings.items()}
mean_core_ms = timing_metrics["all_coco_core_ms"]["mean"]
metrics = {
"frames": {"requested": 11, "completed": totals["frame_count"]},
"inference_passes_per_frame": 1,
"frozen_detection_count": totals["frozen_detection_count"],
"all_coco_detection_count": totals["all_coco_detection_count"],
"added_detection_count": totals["added_detection_count"],
"all_coco_class_counts": {
key.removeprefix("class:"): value
for key, value in sorted(totals.items())
if key.startswith("class:")
},
"timing_ms": timing_metrics,
"postprocess_benchmark": {
"iterations_per_profile_per_frame": POSTPROCESS_BENCHMARK_ITERATIONS,
"timing_ms": {
name: _timing_summary(values)
for name, values in postprocess_benchmark.items()
},
},
"all_coco_core_capacity_fps": round(1000.0 / mean_core_ms, 6),
"authority": AUTHORITY,
}
identity = {
"schema_version": SCHEMA,
"model_sha256": YOLOX_MODEL_SHA256,
"class_count": len(COCO_CLASSES),
"frozen_target_class_ids": list(FROZEN_YOLOX_CONFIG.target_class_ids),
"all_coco_target_class_ids": list(ALL_COCO_YOLOX_CONFIG.target_class_ids),
"valid_fov_mask_sha256": _sha256(arguments.valid_fov_mask),
"producer_sha256": _sha256(Path(__file__)),
"frames_sha256": hashlib.sha256(frame_bytes).hexdigest(),
"metrics": metrics,
"completed": totals["frame_count"] == 11,
"accepted": False,
"authority": AUTHORITY,
}
result_id = "m48s-yolox-all-coco-shadow-" + hashlib.sha256(
_canonical_json(identity)
).hexdigest()
destination = output_parent / result_id
if destination.exists():
raise RuntimeError("immutable M48S all-COCO result already exists")
destination.mkdir(mode=0o700)
(destination / "frames.jsonl").write_bytes(frame_bytes)
(destination / "manifest.json").write_bytes(
_canonical_json({"result_id": result_id, **identity}) + b"\n"
)
(destination / "report.json").write_bytes(
_canonical_json(
{
"schema_version": SCHEMA,
"result_id": result_id,
"completed": identity["completed"],
"accepted": False,
"metrics": metrics,
"decision": {
"all_coco_emission_completed": True,
"additional_inference_passes": 0,
"navigation_or_safety_accepted": False,
"next_gate": "full-load all-COCO detector replay",
},
"authority": AUTHORITY,
}
)
+ b"\n"
)
print(result_id)
print(json.dumps(metrics, indent=2, sort_keys=True))
return 0
def _detection_identity(
class_id: int,
score: float,
box: tuple[float, float, float, float],
) -> tuple[int, float, tuple[float, float, float, float]]:
return class_id, score, box
def _benchmark_postprocess(
output: np.ndarray[Any, Any],
mask: np.ndarray[Any, Any],
*,
destination: dict[str, list[float]],
) -> None:
for iteration in range(POSTPROCESS_BENCHMARK_ITERATIONS):
profiles: tuple[tuple[str, YoloxPostprocessConfig], ...]
if iteration % 2:
profiles = (
("frozen_ms", FROZEN_YOLOX_CONFIG),
("all_coco_ms", ALL_COCO_YOLOX_CONFIG),
)
else:
profiles = (
("all_coco_ms", ALL_COCO_YOLOX_CONFIG),
("frozen_ms", FROZEN_YOLOX_CONFIG),
)
for name, profile in profiles:
started = time.perf_counter_ns()
postprocess_yolox(output, mask, config=profile)
completed = time.perf_counter_ns()
destination[name].append(_milliseconds(started, completed))
def _detection_document(item: Any) -> dict[str, object]:
return {
"class_id": item.class_id,
"label": item.label,
"score": item.score,
"bbox_xyxy": list(item.bbox_xyxy),
"valid_fov_fraction": item.valid_fov_fraction,
}
def _milliseconds(started: int, completed: int) -> float:
return round(max(0, completed - started) / 1_000_000.0, 6)
def _timing_summary(values: list[float]) -> dict[str, float]:
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"max": round(float(array.max()), 6),
}
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_m48s_fixed_detector_candidate_worker.py
+546
View File
@@ -0,0 +1,546 @@
#!/usr/bin/env python3
"""Run one fixed-class detector candidate on the exact M48S risk slice."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
import time
from collections import Counter
from pathlib import Path
from typing import Any, Final, Protocol, cast
import numpy as np
import torch # type: ignore[import-not-found]
from PIL import Image, ImageDraw
WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0"
EXACT_FRAME_NAMES: Final = (
"frame-000121.png",
"frame-000131.png",
"frame-000253.png",
"frame-000275.png",
"frame-000443.png",
"frame-000463.png",
"frame-001094.png",
"frame-001228.png",
"frame-001454.png",
"frame-001856.png",
"frame-002386.png",
)
COCO_CLASSES: Final = (
"person",
"bicycle",
"car",
"motorcycle",
"airplane",
"bus",
"train",
"truck",
"boat",
"traffic light",
"fire hydrant",
"stop sign",
"parking meter",
"bench",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"backpack",
"umbrella",
"handbag",
"tie",
"suitcase",
"frisbee",
"skis",
"snowboard",
"sports ball",
"kite",
"baseball bat",
"baseball glove",
"skateboard",
"surfboard",
"tennis racket",
"bottle",
"wine glass",
"cup",
"fork",
"knife",
"spoon",
"bowl",
"banana",
"apple",
"sandwich",
"orange",
"broccoli",
"carrot",
"hot dog",
"pizza",
"donut",
"cake",
"chair",
"couch",
"potted plant",
"bed",
"dining table",
"toilet",
"tv",
"laptop",
"mouse",
"remote",
"keyboard",
"cell phone",
"microwave",
"oven",
"toaster",
"sink",
"refrigerator",
"book",
"clock",
"vase",
"scissors",
"teddy bear",
"hair drier",
"toothbrush",
)
AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
MINIMUM_BOX_AREA_PIXELS: Final = 64.0
MAXIMUM_BOX_AREA_FRACTION: Final = 0.5
MINIMUM_VALID_FOV_FRACTION: Final = 0.5
OVERLAY_THRESHOLD: Final = 0.25
class Detector(Protocol):
def infer(self, image: Image.Image) -> tuple[RawDetection, ...]: ...
class RawDetection(tuple[int, str, float, tuple[float, float, float, float]]):
"""Normalized detector output: class id, label, score and source-pixel box."""
__slots__ = ()
def __new__(
cls,
class_id: int,
label: str,
score: float,
box: tuple[float, float, float, float],
) -> RawDetection:
return tuple.__new__(cls, (class_id, label, score, box))
@property
def class_id(self) -> int:
return self[0]
@property
def label(self) -> str:
return self[1]
@property
def score(self) -> float:
return self[2]
@property
def box(self) -> tuple[float, float, float, float]:
return self[3]
class DfineDetector:
"""Pinned D-FINE-S COCO PyTorch qualification adapter."""
def __init__(self, source_root: Path, config_path: Path, checkpoint: Path) -> None:
sys.path.insert(0, str(source_root))
from src.core import YAMLConfig # type: ignore[import-not-found]
config = YAMLConfig(str(config_path), resume=str(checkpoint))
if "HGNetv2" in config.yaml_cfg:
config.yaml_cfg["HGNetv2"]["pretrained"] = False
state = torch.load(checkpoint, map_location="cpu", weights_only=True)
weights = state["ema"]["module"] if "ema" in state else state["model"]
config.model.load_state_dict(weights)
self._model = config.model.deploy().to("cuda").eval()
self._postprocessor = config.postprocessor.deploy()
def infer(self, image: Image.Image) -> tuple[RawDetection, ...]:
tensor, ratio, padding = _dfine_preprocess(image)
with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16):
output = self._model(tensor)
size = torch.tensor([[640, 640]], device="cuda")
labels, boxes, scores = self._postprocessor(output, size)
labels_array = labels[0].detach().to("cpu").numpy()
boxes_array = boxes[0].detach().to("cpu").numpy()
scores_array = scores[0].detach().to("cpu").numpy()
pad_x, pad_y = padding
detections = []
for raw_label, raw_score, raw_box in zip(
labels_array,
scores_array,
boxes_array,
strict=True,
):
class_id = int(raw_label)
if not 0 <= class_id < len(COCO_CLASSES):
continue
box = (
(float(raw_box[0]) - pad_x) / ratio,
(float(raw_box[1]) - pad_y) / ratio,
(float(raw_box[2]) - pad_x) / ratio,
(float(raw_box[3]) - pad_y) / ratio,
)
detections.append(
RawDetection(class_id, COCO_CLASSES[class_id], float(raw_score), box)
)
return tuple(detections)
class RfDetrDetector:
"""Pinned RF-DETR-L COCO PyTorch qualification adapter."""
def __init__(self, checkpoint: Path) -> None:
from rfdetr import RFDETRLarge # type: ignore[import-not-found]
self._model = RFDETRLarge(pretrain_weights=str(checkpoint))
self._model.inference(compile=False, dtype=torch.float16, inplace=True)
def infer(self, image: Image.Image) -> tuple[RawDetection, ...]:
prediction = self._model.predict(
image,
threshold=0.1,
include_source_image=False,
)
boxes = np.asarray(prediction.xyxy)
scores = np.asarray(prediction.confidence)
names = np.asarray(prediction.data["class_name"])
detections = []
for raw_name, raw_score, raw_box in zip(names, scores, boxes, strict=True):
label = str(raw_name)
try:
class_id = COCO_CLASSES.index(label)
except ValueError:
continue
detections.append(
RawDetection(
class_id,
label,
float(raw_score),
cast(
tuple[float, float, float, float],
tuple(float(value) for value in raw_box),
),
)
)
return tuple(detections)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--candidate",
choices=("dfine-s-coco", "rf-detr-large-coco"),
required=True,
)
parser.add_argument("--profile-id", required=True)
parser.add_argument("--provider-id", required=True)
parser.add_argument("--upstream-revision", required=True)
parser.add_argument("--input-root", type=Path, required=True)
parser.add_argument("--valid-fov-mask", type=Path, required=True)
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--expected-checkpoint-sha256", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--dfine-root", type=Path, default=Path("/opt/dfine"))
parser.add_argument(
"--dfine-config",
type=Path,
default=Path("/opt/dfine/configs/dfine/dfine_hgnetv2_s_coco.yml"),
)
parser.add_argument("--warmup-iterations", type=int, default=5)
parser.add_argument("--benchmark-iterations", type=int, default=30)
arguments = parser.parse_args()
if arguments.warmup_iterations < 1 or arguments.benchmark_iterations < 1:
raise RuntimeError("warmup and benchmark iterations must be positive")
input_root = arguments.input_root.resolve(strict=True)
images = tuple(sorted(input_root.glob("frame-*.png")))
if tuple(path.name for path in images) != EXACT_FRAME_NAMES:
raise RuntimeError("candidate Worker requires the exact M48S risk slice")
mask = _load_mask(arguments.valid_fov_mask.resolve(strict=True))
checkpoint = arguments.checkpoint.resolve(strict=True)
checkpoint_sha256 = _sha256(checkpoint)
if checkpoint_sha256 != arguments.expected_checkpoint_sha256:
raise RuntimeError("checkpoint SHA-256 does not match the pinned profile")
output = arguments.output.absolute()
if output.exists():
raise RuntimeError("candidate Worker output already exists")
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
overlay_root = output.parent / f"{output.stem}-overlays"
overlay_root.mkdir(mode=0o700)
gpu_before = _gpu_sample()
started_utc_ns = time.time_ns()
torch.cuda.reset_peak_memory_stats()
detector: Detector
if arguments.candidate == "dfine-s-coco":
detector = DfineDetector(
arguments.dfine_root.resolve(strict=True),
arguments.dfine_config.resolve(strict=True),
checkpoint,
)
else:
detector = RfDetrDetector(checkpoint)
with Image.open(images[0]) as opened:
warmup_image = opened.convert("RGB")
for _ in range(arguments.warmup_iterations):
detector.infer(warmup_image)
_synchronize()
frames = []
totals: Counter[str] = Counter()
evidence_timings = []
for image_path in images:
with Image.open(image_path) as opened:
image = opened.convert("RGB")
_synchronize()
started = time.perf_counter_ns()
raw = detector.infer(image)
_synchronize()
completed = time.perf_counter_ns()
elapsed_ms = _milliseconds(started, completed)
evidence_timings.append(elapsed_ms)
detections = _qualify(raw, mask, image.size)
totals["frame_count"] += 1
totals["detection_count"] += len(detections)
for detection in detections:
totals[f"class:{detection['label']}"] += 1
frames.append(
{
"frame_name": image_path.name,
"source_sha256": _sha256(image_path),
"detections": detections,
"timing_ms": {"end_to_end": elapsed_ms},
"authority": AUTHORITY,
}
)
_write_overlay(image, detections, overlay_root / image_path.name)
with Image.open(input_root / "frame-000253.png") as opened:
benchmark_image = opened.convert("RGB")
benchmark_timings = []
for _ in range(arguments.benchmark_iterations):
_synchronize()
started = time.perf_counter_ns()
detector.infer(benchmark_image)
_synchronize()
benchmark_timings.append(_milliseconds(started, time.perf_counter_ns()))
gpu_after = _gpu_sample()
timing = _timing_summary(evidence_timings)
benchmark_timing = _timing_summary(benchmark_timings)
metrics = {
"frames": {"requested": 11, "completed": totals["frame_count"]},
"detection_count_at_minimum_score_0_1": totals["detection_count"],
"class_counts_at_minimum_score_0_1": {
key.removeprefix("class:"): value
for key, value in sorted(totals.items())
if key.startswith("class:")
},
"evidence_timing_ms": timing,
"benchmark": {
"frame_name": "frame-000253.png",
"iterations": arguments.benchmark_iterations,
"timing_ms": benchmark_timing,
"core_capacity_fps": round(1000.0 / benchmark_timing["mean"], 6),
},
"torch_peak_memory": {
"allocated_bytes": torch.cuda.max_memory_allocated(),
"reserved_bytes": torch.cuda.max_memory_reserved(),
},
"gpu_before": gpu_before,
"gpu_after": gpu_after,
}
document = {
"schema_version": WORKER_RUN_SCHEMA,
"profile_id": arguments.profile_id,
"provider_id": arguments.provider_id,
"candidate": arguments.candidate,
"upstream_revision": arguments.upstream_revision,
"checkpoint_sha256": checkpoint_sha256,
"started_utc_ns": started_utc_ns,
"completed_utc_ns": time.time_ns(),
"completed": totals["frame_count"] == 11,
"execution": {
"worker_id": "worker-006",
"device": torch.cuda.get_device_name(0),
"precision": "fp16",
"inference_passes_per_evidence_frame": 1,
"warmup_iterations": arguments.warmup_iterations,
"benchmark_iterations": arguments.benchmark_iterations,
"concurrent_services_retained": True,
},
"frames": frames,
"metrics": metrics,
"authority": AUTHORITY,
}
output.write_bytes(_canonical_json(document) + b"\n")
print(output)
print(json.dumps(metrics, indent=2, sort_keys=True))
return 0
def _dfine_preprocess(
image: Image.Image,
) -> tuple[torch.Tensor, float, tuple[int, int]]:
width, height = image.size
ratio = min(640.0 / width, 640.0 / height)
resized_width = int(width * ratio)
resized_height = int(height * ratio)
resized = image.resize((resized_width, resized_height), Image.Resampling.BILINEAR)
padded = Image.new("RGB", (640, 640))
pad_x = (640 - resized_width) // 2
pad_y = (640 - resized_height) // 2
padded.paste(resized, (pad_x, pad_y))
array = np.asarray(padded, dtype=np.float32) / 255.0
tensor = torch.from_numpy(np.ascontiguousarray(array.transpose(2, 0, 1)))
return tensor.unsqueeze(0).to("cuda", non_blocking=True), ratio, (pad_x, pad_y)
def _qualify(
detections: tuple[RawDetection, ...],
mask: np.ndarray[Any, Any],
image_size: tuple[int, int],
) -> list[dict[str, object]]:
width, height = image_size
image_area = float(width * height)
qualified = []
for detection in detections:
if detection.score < 0.1:
continue
x1, y1, x2, y2 = detection.box
x1 = max(0.0, min(float(width), x1))
y1 = max(0.0, min(float(height), y1))
x2 = max(0.0, min(float(width), x2))
y2 = max(0.0, min(float(height), y2))
area = max(0.0, x2 - x1) * max(0.0, y2 - y1)
if area < MINIMUM_BOX_AREA_PIXELS or area > MAXIMUM_BOX_AREA_FRACTION * image_area:
continue
center_x = min(width - 1, max(0, int((x1 + x2) / 2.0)))
center_y = min(height - 1, max(0, int((y1 + y2) / 2.0)))
if not bool(mask[center_y, center_x]):
continue
ix1 = min(width - 1, max(0, int(np.floor(x1))))
iy1 = min(height - 1, max(0, int(np.floor(y1))))
ix2 = min(width, max(ix1 + 1, int(np.ceil(x2))))
iy2 = min(height, max(iy1 + 1, int(np.ceil(y2))))
valid_fraction = float(mask[iy1:iy2, ix1:ix2].mean())
if valid_fraction < MINIMUM_VALID_FOV_FRACTION:
continue
qualified.append(
{
"class_id": detection.class_id,
"label": detection.label,
"score": round(detection.score, 6),
"bbox_xyxy": [round(value, 3) for value in (x1, y1, x2, y2)],
"valid_fov_fraction": round(valid_fraction, 6),
}
)
qualified.sort(key=lambda item: (-cast(float, item["score"]), cast(int, item["class_id"])))
return qualified
def _write_overlay(
image: Image.Image,
detections: list[dict[str, object]],
destination: Path,
) -> None:
annotated = image.copy()
draw = ImageDraw.Draw(annotated)
for detection in detections:
score = cast(float, detection["score"])
if score < OVERLAY_THRESHOLD:
continue
box = cast(list[float], detection["bbox_xyxy"])
label = cast(str, detection["label"])
draw.rectangle(box, outline=(255, 84, 0), width=3)
draw.text((box[0] + 3, box[1] + 3), f"{label} {score:.2f}", fill=(255, 255, 255))
annotated.save(destination)
def _load_mask(path: Path) -> np.ndarray[Any, Any]:
with Image.open(path) as image:
array = np.asarray(image.convert("L"), dtype=np.uint8)
if array.shape != (600, 800):
raise RuntimeError("valid-FOV mask must be 800x600")
return array > 0
def _gpu_sample() -> dict[str, object]:
completed = subprocess.run(
[
"nvidia-smi",
"--query-gpu=name,memory.total,memory.used,utilization.gpu,temperature.gpu,power.draw",
"--format=csv,noheader,nounits",
],
check=True,
capture_output=True,
text=True,
)
values = [value.strip() for value in completed.stdout.strip().split(",")]
if len(values) != 6:
raise RuntimeError("unexpected nvidia-smi response")
return {
"name": values[0],
"memory_total_mib": float(values[1]),
"memory_used_mib": float(values[2]),
"utilization_gpu_percent": float(values[3]),
"temperature_c": float(values[4]),
"power_w": float(values[5]),
}
def _synchronize() -> None:
torch.cuda.synchronize()
def _timing_summary(values: list[float]) -> dict[str, float]:
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"max": round(float(array.max()), 6),
}
def _milliseconds(started: int, completed: int) -> float:
return round(max(0, completed - started) / 1_000_000.0, 6)
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_m48s_rf_detr_load_worker.py
+417
View File
@@ -0,0 +1,417 @@
#!/usr/bin/env python3
"""Run a source-paced bounded-queue RF-DETR/Triton stability qualification."""
from __future__ import annotations
import argparse
import hashlib
import json
import resource
import subprocess
import threading
import time
from collections import Counter, deque
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final
import av # type: ignore[import-not-found]
import numpy as np
import run_rf_detr_triton as qualifier # type: ignore[import-not-found]
import tritonclient.http as httpclient # type: ignore[import-not-found]
from PIL import Image
SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-source-paced-load/v0"
SOURCE_SHA256: Final = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8"
SOURCE_FPS: Final = 10.003944527024467
AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
@dataclass(frozen=True, slots=True)
class SourceFrame:
sequence: int
scheduled_ns: int
image: Image.Image
class LatestWinsQueue:
"""Bounded source queue that reports every replacement explicitly."""
def __init__(self, capacity: int) -> None:
if capacity < 1:
raise ValueError("queue capacity must be positive")
self.capacity = capacity
self._items: deque[SourceFrame] = deque()
self._condition = threading.Condition()
self._closed = False
self.replacements = 0
self.maximum_depth = 0
def put(self, item: SourceFrame) -> None:
with self._condition:
if self._closed:
return
if len(self._items) == self.capacity:
self._items.popleft()
self.replacements += 1
self._items.append(item)
self.maximum_depth = max(self.maximum_depth, len(self._items))
self._condition.notify()
def get(self) -> SourceFrame | None:
with self._condition:
while not self._items and not self._closed:
self._condition.wait(timeout=1.0)
if self._items:
return self._items.popleft()
return None
def close(self) -> None:
with self._condition:
self._closed = True
self._condition.notify_all()
class GpuTelemetry:
def __init__(self, interval_seconds: float) -> None:
self.interval_seconds = interval_seconds
self.samples: list[dict[str, float]] = []
self._stop = threading.Event()
self._thread = threading.Thread(target=self._run, daemon=True)
def __enter__(self) -> GpuTelemetry:
self._thread.start()
return self
def __exit__(self, *_args: object) -> None:
self._stop.set()
self._thread.join(timeout=10.0)
def _run(self) -> None:
while not self._stop.is_set():
try:
completed = subprocess.run(
[
"nvidia-smi",
"--query-gpu=utilization.gpu,memory.used,power.draw,temperature.gpu",
"--format=csv,noheader,nounits",
],
check=True,
capture_output=True,
text=True,
timeout=10.0,
)
values = [float(value.strip()) for value in completed.stdout.split(",")]
if len(values) == 4:
self.samples.append(
{
"gpu_utilization_percent": values[0],
"gpu_memory_used_mib": values[1],
"gpu_power_w": values[2],
"gpu_temperature_c": values[3],
}
)
except (OSError, ValueError, subprocess.SubprocessError):
pass
self._stop.wait(self.interval_seconds)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--source-video", type=Path, required=True)
parser.add_argument("--valid-fov-mask", type=Path, required=True)
parser.add_argument("--endpoint", default="localhost:8100")
parser.add_argument("--model-name", default="rf_detr_large")
parser.add_argument("--duration-seconds", type=float, default=1800.0)
parser.add_argument("--queue-capacity", type=int, default=2)
parser.add_argument("--telemetry-interval-seconds", type=float, default=1.0)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--progress", type=Path, required=True)
arguments = parser.parse_args()
if arguments.duration_seconds <= 0:
raise RuntimeError("duration must be positive")
if arguments.telemetry_interval_seconds <= 0:
raise RuntimeError("telemetry interval must be positive")
source = arguments.source_video.resolve(strict=True)
if _sha256(source) != SOURCE_SHA256:
raise RuntimeError("RAVNOVES00 camera stream identity changed")
mask = qualifier._load_mask(arguments.valid_fov_mask.resolve(strict=True))
output = arguments.output.absolute()
progress = arguments.progress.absolute()
if output.exists() or progress.exists():
raise RuntimeError("load result or progress artifact already exists")
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
client = httpclient.InferenceServerClient(arguments.endpoint, concurrency=1)
if not client.is_server_ready() or not client.is_model_ready(arguments.model_name):
raise RuntimeError("isolated Triton or RF-DETR model is not ready")
metadata = client.get_model_metadata(arguments.model_name)
qualifier._validate_metadata(metadata)
triton_before = client.get_inference_statistics(arguments.model_name)
gpu_before = qualifier._gpu_sample()
rss_before_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
queue = LatestWinsQueue(arguments.queue_capacity)
producer_stop = threading.Event()
producer_errors: list[str] = []
produced_count = [0]
source_loops = [0]
started_ns = time.monotonic_ns()
started_utc_ns = time.time_ns()
producer = threading.Thread(
target=_produce,
args=(
source,
queue,
producer_stop,
producer_errors,
produced_count,
source_loops,
started_ns,
arguments.duration_seconds,
),
daemon=True,
)
consumed = 0
failures = 0
class_counts: Counter[str] = Counter()
end_to_end_ms: list[float] = []
triton_round_trip_ms: list[float] = []
completion_age_ms: list[float] = []
last_progress_ns = started_ns
producer.start()
with progress.open("x", encoding="utf-8") as progress_stream, GpuTelemetry(
arguments.telemetry_interval_seconds
) as telemetry:
try:
while True:
item = queue.get()
if item is None:
break
raw, timing = qualifier._infer(client, arguments.model_name, item.image)
detections = qualifier._qualify(raw, mask, item.image.size)
for detection in detections:
if float(detection["score"]) >= 0.5:
class_counts[str(detection["label"])] += 1
consumed += 1
end_to_end_ms.append(timing["end_to_end"])
triton_round_trip_ms.append(timing["triton_round_trip"])
completion_age_ms.append((time.monotonic_ns() - item.scheduled_ns) / 1_000_000.0)
now_ns = time.monotonic_ns()
if now_ns - last_progress_ns >= 60_000_000_000:
row = {
"elapsed_seconds": round((now_ns - started_ns) / 1_000_000_000.0, 3),
"produced": produced_count[0],
"consumed": consumed,
"replacements": queue.replacements,
"completion_age_p95_ms": _distribution(completion_age_ms)["p95"],
}
progress_stream.write(json.dumps(row, separators=(",", ":")) + "\n")
progress_stream.flush()
print(json.dumps(row, sort_keys=True), flush=True)
last_progress_ns = now_ns
except BaseException:
failures += 1
raise
finally:
producer_stop.set()
queue.close()
producer.join(timeout=15.0)
completed_ns = time.monotonic_ns()
wall_seconds = (completed_ns - started_ns) / 1_000_000_000.0
if producer.is_alive():
raise RuntimeError("source producer did not stop")
if producer_errors:
raise RuntimeError(f"source producer failed: {producer_errors}")
triton_after = client.get_inference_statistics(arguments.model_name)
gpu_after = qualifier._gpu_sample()
rss_after_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
core = _distribution(end_to_end_ms)
completion_age = _distribution(completion_age_ms)
telemetry_summary = _telemetry_summary(telemetry.samples)
checks = {
"minimum_duration": wall_seconds >= arguments.duration_seconds,
"zero_failures": failures == 0,
"minimum_source_fps": consumed / arguments.duration_seconds >= 9.5,
"maximum_detector_completion_age_p95_ms": completion_age["p95"] <= 175.0,
"maximum_worker_vram_gib": (
float(telemetry_summary["gpu_memory_used_mib"]["maximum"]) <= 20 * 1024
),
"no_sustained_100_percent_gpu": _longest_full_gpu_run(telemetry.samples)
< max(5, round(30.0 / arguments.telemetry_interval_seconds)),
"bounded_latest_wins_queue": queue.maximum_depth <= arguments.queue_capacity,
"triton_request_accounting": _triton_inference_count(triton_after)
- _triton_inference_count(triton_before)
== consumed,
}
detector_load_gate_passed = all(checks.values())
document = {
"schema_version": SCHEMA_VERSION,
"profile_id": "rf-detr-large-coco-704-trt11-fp16-source-paced/v0",
"source": {
"source_id": "RAVNOVES00",
"sha256": SOURCE_SHA256,
"frame_rate": SOURCE_FPS,
"duration_seconds": arguments.duration_seconds,
"source_loops": source_loops[0],
},
"model_metadata": metadata,
"execution": {
"worker_id": "worker-006",
"queue_policy": "bounded-latest-wins",
"queue_capacity": arguments.queue_capacity,
"queue_maximum_depth": queue.maximum_depth,
"source_frames_produced": produced_count[0],
"source_frames_consumed": consumed,
"source_frame_replacements": queue.replacements,
"failures": failures,
"wall_seconds": round(wall_seconds, 6),
"effective_consumed_fps": round(consumed / arguments.duration_seconds, 6),
"background_services_retained": True,
},
"metrics": {
"end_to_end_ms": core,
"triton_round_trip_ms": _distribution(triton_round_trip_ms),
"detector_completion_age_ms": completion_age,
"class_counts_at_score_0_5": dict(sorted(class_counts.items())),
"gpu": telemetry_summary,
"gpu_before": gpu_before,
"gpu_after": gpu_after,
"process_peak_rss_before_mib": round(rss_before_kib / 1024.0, 6),
"process_peak_rss_after_mib": round(rss_after_kib / 1024.0, 6),
"triton_statistics_before": triton_before,
"triton_statistics_after": triton_after,
},
"checks": checks,
"detector_load_gate_passed": detector_load_gate_passed,
"integrated_world_state_gate_evaluated": False,
"candidate_accepted": False,
"started_utc_ns": started_utc_ns,
"completed_utc_ns": time.time_ns(),
"completed": True,
"authority": AUTHORITY,
}
output.write_bytes(_canonical_json(document) + b"\n")
print(output)
print(json.dumps(document["execution"], indent=2, sort_keys=True))
print(json.dumps(checks, indent=2, sort_keys=True))
return 0 if detector_load_gate_passed else 2
def _produce(
source: Path,
queue: LatestWinsQueue,
stop: threading.Event,
errors: list[str],
produced_count: list[int],
source_loops: list[int],
started_ns: int,
duration_seconds: float,
) -> None:
try:
period_ns = round(1_000_000_000.0 / SOURCE_FPS)
while not stop.is_set():
container = av.open(str(source))
try:
streams = container.streams.video
if len(streams) != 1:
raise RuntimeError("RAVNOVES00 video stream count changed")
for decoded in container.decode(streams[0]):
sequence = produced_count[0]
scheduled_ns = started_ns + sequence * period_ns
if scheduled_ns - started_ns >= round(duration_seconds * 1_000_000_000):
queue.close()
return
remaining_seconds = (scheduled_ns - time.monotonic_ns()) / 1_000_000_000.0
if remaining_seconds > 0 and stop.wait(remaining_seconds):
queue.close()
return
bgr = decoded.to_ndarray(format="bgr24")
if bgr.shape != (600, 800, 3):
raise RuntimeError("RAVNOVES00 source raster changed")
rgb = np.ascontiguousarray(bgr[:, :, ::-1])
queue.put(SourceFrame(sequence, scheduled_ns, Image.fromarray(rgb, "RGB")))
produced_count[0] += 1
if stop.is_set():
queue.close()
return
source_loops[0] += 1
finally:
container.close()
except BaseException as error:
errors.append(f"{type(error).__name__}: {error}")
queue.close()
def _distribution(values: list[float]) -> dict[str, float]:
if not values:
return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "maximum": 0.0}
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"maximum": round(float(array.max()), 6),
}
def _telemetry_summary(samples: list[dict[str, float]]) -> dict[str, Any]:
summary: dict[str, Any] = {"sample_count": len(samples)}
for key in (
"gpu_utilization_percent",
"gpu_memory_used_mib",
"gpu_power_w",
"gpu_temperature_c",
):
summary[key] = _distribution([sample[key] for sample in samples])
summary["longest_100_percent_gpu_sample_run"] = _longest_full_gpu_run(samples)
return summary
def _longest_full_gpu_run(samples: list[dict[str, float]]) -> int:
longest = 0
current = 0
for sample in samples:
if sample["gpu_utilization_percent"] >= 100.0:
current += 1
longest = max(longest, current)
else:
current = 0
return longest
def _triton_inference_count(statistics: dict[str, Any]) -> int:
model_stats = statistics.get("model_stats")
if not isinstance(model_stats, list) or len(model_stats) != 1:
raise RuntimeError("unexpected Triton model statistics")
count = model_stats[0].get("inference_count")
if not isinstance(count, int):
raise RuntimeError("Triton inference count is unavailable")
return count
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/run_m48s_rf_detr_triton_worker.py
+508
View File
@@ -0,0 +1,508 @@
#!/usr/bin/env python3
"""Qualify the pinned RF-DETR TensorRT finalist through isolated Triton."""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import time
from collections import Counter
from pathlib import Path
from typing import Any, Final, cast
import numpy as np
import tritonclient.http as httpclient # type: ignore[import-not-found]
from PIL import Image, ImageDraw
from torchvision.transforms import functional as vision_functional # type: ignore[import-not-found]
WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0"
EXACT_FRAME_NAMES: Final = (
"frame-000121.png",
"frame-000131.png",
"frame-000253.png",
"frame-000275.png",
"frame-000443.png",
"frame-000463.png",
"frame-001094.png",
"frame-001228.png",
"frame-001454.png",
"frame-001856.png",
"frame-002386.png",
)
COCO_CLASSES: Final = (
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck",
"boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
"giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
"skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
"skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli",
"carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant",
"bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard",
"cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book",
"clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush",
)
COCO_SPARSE_IDS: Final = (
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42,
43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,
61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81,
82, 84, 85, 86, 87, 88, 89, 90,
)
COCO_SPARSE_NAMES: Final = dict(zip(COCO_SPARSE_IDS, COCO_CLASSES, strict=True))
AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
MEANS: Final = (0.485, 0.456, 0.406)
STDS: Final = (0.229, 0.224, 0.225)
MINIMUM_BOX_AREA_PIXELS: Final = 64.0
MAXIMUM_BOX_AREA_FRACTION: Final = 0.5
MINIMUM_VALID_FOV_FRACTION: Final = 0.5
OVERLAY_THRESHOLD: Final = 0.25
class RawDetection(tuple[int, str, float, tuple[float, float, float, float]]):
__slots__ = ()
def __new__(
cls,
class_id: int,
label: str,
score: float,
box: tuple[float, float, float, float],
) -> RawDetection:
return tuple.__new__(cls, (class_id, label, score, box))
@property
def class_id(self) -> int:
return self[0]
@property
def label(self) -> str:
return self[1]
@property
def score(self) -> float:
return self[2]
@property
def box(self) -> tuple[float, float, float, float]:
return self[3]
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--endpoint", default="localhost:8100")
parser.add_argument("--model-name", default="rf_detr_large")
parser.add_argument("--profile-id", required=True)
parser.add_argument("--provider-id", required=True)
parser.add_argument("--upstream-revision", required=True)
parser.add_argument("--input-root", type=Path, required=True)
parser.add_argument("--valid-fov-mask", type=Path, required=True)
parser.add_argument("--engine", type=Path, required=True)
parser.add_argument("--expected-engine-sha256", required=True)
parser.add_argument("--checkpoint-sha256", required=True)
parser.add_argument("--pytorch-reference", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--warmup-iterations", type=int, default=10)
parser.add_argument("--benchmark-iterations", type=int, default=100)
arguments = parser.parse_args()
if arguments.warmup_iterations < 1 or arguments.benchmark_iterations < 1:
raise RuntimeError("warmup and benchmark iterations must be positive")
input_root = arguments.input_root.resolve(strict=True)
images = tuple(sorted(input_root.glob("frame-*.png")))
if tuple(path.name for path in images) != EXACT_FRAME_NAMES:
raise RuntimeError("Triton qualifier requires the exact M48S risk slice")
mask = _load_mask(arguments.valid_fov_mask.resolve(strict=True))
engine = arguments.engine.resolve(strict=True)
engine_sha256 = _sha256(engine)
if engine_sha256 != arguments.expected_engine_sha256:
raise RuntimeError("TensorRT engine SHA-256 does not match the pinned finalist")
reference_path = arguments.pytorch_reference.resolve(strict=True)
output = arguments.output.absolute()
if output.exists():
raise RuntimeError("Triton Worker output already exists")
output.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
overlay_root = output.parent / f"{output.stem}-overlays"
overlay_root.mkdir(mode=0o700)
client = httpclient.InferenceServerClient(arguments.endpoint, concurrency=1)
if not client.is_server_ready() or not client.is_model_ready(arguments.model_name):
raise RuntimeError("isolated Triton or RF-DETR model is not ready")
metadata = client.get_model_metadata(arguments.model_name)
_validate_metadata(metadata)
statistics_before = client.get_inference_statistics(arguments.model_name)
gpu_before = _gpu_sample()
started_utc_ns = time.time_ns()
with Image.open(images[0]) as opened:
warmup_image = opened.convert("RGB")
for _ in range(arguments.warmup_iterations):
_infer(client, arguments.model_name, warmup_image)
frames = []
totals: Counter[str] = Counter()
evidence_timings = []
network_timings = []
for image_path in images:
with Image.open(image_path) as opened:
image = opened.convert("RGB")
raw, timing = _infer(client, arguments.model_name, image)
evidence_timings.append(timing["end_to_end"])
network_timings.append(timing["triton_round_trip"])
detections = _qualify(raw, mask, image.size)
totals["frame_count"] += 1
totals["detection_count"] += len(detections)
for detection in detections:
totals[f"class:{detection['label']}"] += 1
frames.append(
{
"frame_name": image_path.name,
"source_sha256": _sha256(image_path),
"detections": detections,
"timing_ms": timing,
"authority": AUTHORITY,
}
)
_write_overlay(image, detections, overlay_root / image_path.name)
with Image.open(input_root / "frame-000253.png") as opened:
benchmark_image = opened.convert("RGB")
benchmark_timings = []
benchmark_network_timings = []
for _ in range(arguments.benchmark_iterations):
_, timing = _infer(client, arguments.model_name, benchmark_image)
benchmark_timings.append(timing["end_to_end"])
benchmark_network_timings.append(timing["triton_round_trip"])
gpu_after = _gpu_sample()
statistics_after = client.get_inference_statistics(arguments.model_name)
benchmark = _timing_summary(benchmark_timings)
parity = _reference_parity(frames, reference_path)
metrics = {
"frames": {"requested": 11, "completed": totals["frame_count"]},
"detection_count_at_minimum_score_0_1": totals["detection_count"],
"class_counts_at_minimum_score_0_1": {
key.removeprefix("class:"): value
for key, value in sorted(totals.items())
if key.startswith("class:")
},
"evidence_timing_ms": _timing_summary(evidence_timings),
"evidence_triton_round_trip_ms": _timing_summary(network_timings),
"benchmark": {
"frame_name": "frame-000253.png",
"iterations": arguments.benchmark_iterations,
"timing_ms": benchmark,
"triton_round_trip_ms": _timing_summary(benchmark_network_timings),
"end_to_end_capacity_fps": round(1000.0 / benchmark["mean"], 6),
},
"pytorch_reference_parity": parity,
"triton_statistics_before": statistics_before,
"triton_statistics_after": statistics_after,
"gpu_before": gpu_before,
"gpu_after": gpu_after,
}
document = {
"schema_version": WORKER_RUN_SCHEMA,
"profile_id": arguments.profile_id,
"provider_id": arguments.provider_id,
"candidate": "rf-detr-large-coco-tensorrt",
"upstream_revision": arguments.upstream_revision,
"checkpoint_sha256": arguments.checkpoint_sha256,
"engine_sha256": engine_sha256,
"started_utc_ns": started_utc_ns,
"completed_utc_ns": time.time_ns(),
"completed": totals["frame_count"] == 11,
"execution": {
"worker_id": "worker-006",
"device": gpu_after["name"],
"precision": "strongly-typed-fp16",
"inference_passes_per_evidence_frame": 1,
"warmup_iterations": arguments.warmup_iterations,
"benchmark_iterations": arguments.benchmark_iterations,
"isolated_triton": True,
"concurrent_services_retained": True,
},
"model_metadata": metadata,
"frames": frames,
"metrics": metrics,
"authority": AUTHORITY,
}
output.write_bytes(_canonical_json(document) + b"\n")
print(output)
print(json.dumps(metrics, indent=2, sort_keys=True))
return 0
def _infer(
client: httpclient.InferenceServerClient,
model_name: str,
image: Image.Image,
) -> tuple[tuple[RawDetection, ...], dict[str, float]]:
started = time.perf_counter_ns()
tensor = vision_functional.to_tensor(image)
tensor = vision_functional.resize(tensor, [704, 704], antialias=False)
tensor = vision_functional.normalize(tensor, MEANS, STDS)
batch = np.ascontiguousarray(tensor.unsqueeze(0).numpy(), dtype=np.float32)
preprocessed = time.perf_counter_ns()
infer_input = httpclient.InferInput("input", batch.shape, "FP32")
infer_input.set_data_from_numpy(batch, binary_data=True)
response = client.infer(
model_name,
[infer_input],
outputs=[
httpclient.InferRequestedOutput("dets", binary_data=True),
httpclient.InferRequestedOutput("labels", binary_data=True),
],
)
inferred = time.perf_counter_ns()
boxes = response.as_numpy("dets")
logits = response.as_numpy("labels")
if boxes is None or logits is None:
raise RuntimeError("Triton RF-DETR response is missing outputs")
detections = decode_outputs(boxes, logits, image.size)
completed = time.perf_counter_ns()
return detections, {
"preprocess": _milliseconds(started, preprocessed),
"triton_round_trip": _milliseconds(preprocessed, inferred),
"postprocess": _milliseconds(inferred, completed),
"end_to_end": _milliseconds(started, completed),
}
def decode_outputs(
boxes: np.ndarray[Any, Any],
logits: np.ndarray[Any, Any],
image_size: tuple[int, int],
) -> tuple[RawDetection, ...]:
if boxes.shape != (1, 300, 4) or logits.shape != (1, 300, 91):
raise RuntimeError(f"unexpected RF-DETR output shapes: {boxes.shape}, {logits.shape}")
probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits[0].astype(np.float32), -80.0, 80.0)))
flattened = probabilities.reshape(-1)
topk = np.argsort(-flattened, kind="stable")[:300]
width, height = image_size
decoded = []
for flat_index in topk:
score = float(flattened[flat_index])
if score <= 0.1:
continue
query_index = int(flat_index // logits.shape[2])
sparse_class_id = int(flat_index % logits.shape[2])
label = COCO_SPARSE_NAMES.get(sparse_class_id)
if label is None:
continue
center_x, center_y, box_width, box_height = (
float(value) for value in boxes[0, query_index].astype(np.float32)
)
box = (
(center_x - box_width / 2.0) * width,
(center_y - box_height / 2.0) * height,
(center_x + box_width / 2.0) * width,
(center_y + box_height / 2.0) * height,
)
decoded.append(RawDetection(COCO_CLASSES.index(label), label, score, box))
return tuple(decoded)
def _qualify(
detections: tuple[RawDetection, ...],
mask: np.ndarray[Any, Any],
image_size: tuple[int, int],
) -> list[dict[str, object]]:
width, height = image_size
image_area = float(width * height)
qualified = []
for detection in detections:
x1, y1, x2, y2 = detection.box
x1 = max(0.0, min(float(width), x1))
y1 = max(0.0, min(float(height), y1))
x2 = max(0.0, min(float(width), x2))
y2 = max(0.0, min(float(height), y2))
area = max(0.0, x2 - x1) * max(0.0, y2 - y1)
if area < MINIMUM_BOX_AREA_PIXELS or area > MAXIMUM_BOX_AREA_FRACTION * image_area:
continue
center_x = min(width - 1, max(0, int((x1 + x2) / 2.0)))
center_y = min(height - 1, max(0, int((y1 + y2) / 2.0)))
if not bool(mask[center_y, center_x]):
continue
ix1 = min(width - 1, max(0, int(np.floor(x1))))
iy1 = min(height - 1, max(0, int(np.floor(y1))))
ix2 = min(width, max(ix1 + 1, int(np.ceil(x2))))
iy2 = min(height, max(iy1 + 1, int(np.ceil(y2))))
valid_fraction = float(mask[iy1:iy2, ix1:ix2].mean())
if valid_fraction < MINIMUM_VALID_FOV_FRACTION:
continue
qualified.append(
{
"class_id": detection.class_id,
"label": detection.label,
"score": round(detection.score, 6),
"bbox_xyxy": [round(value, 3) for value in (x1, y1, x2, y2)],
"valid_fov_fraction": round(valid_fraction, 6),
}
)
qualified.sort(key=lambda item: (-cast(float, item["score"]), cast(int, item["class_id"])))
return qualified
def _reference_parity(frames: list[dict[str, object]], reference_path: Path) -> dict[str, object]:
reference = json.loads(reference_path.read_text(encoding="utf-8"))
if not isinstance(reference, dict):
raise RuntimeError("PyTorch reference must be a JSON object")
reference_frames = reference.get("frames")
if not isinstance(reference_frames, list):
raise RuntimeError("PyTorch reference lacks frames")
actual = _best_detection(frames, "frame-000253.png", "dog")
expected = _best_detection(reference_frames, "frame-000253.png", "dog")
if actual is None or expected is None:
return {
"frame_000253_dog_present_in_pytorch": expected is not None,
"frame_000253_dog_present_in_tensorrt": actual is not None,
"score_absolute_delta": None,
"box_iou": None,
"passed": False,
}
score_delta = abs(
float(cast(float, actual["score"])) - float(cast(float, expected["score"]))
)
iou = _box_iou(
cast(list[float], actual["bbox_xyxy"]),
cast(list[float], expected["bbox_xyxy"]),
)
return {
"frame_000253_dog_present_in_pytorch": True,
"frame_000253_dog_present_in_tensorrt": True,
"pytorch_score": expected["score"],
"tensorrt_score": actual["score"],
"score_absolute_delta": round(score_delta, 6),
"box_iou": round(iou, 6),
"passed": score_delta <= 0.05 and iou >= 0.9,
}
def _best_detection(
frames: list[Any], frame_name: str, label: str
) -> dict[str, object] | None:
for frame in frames:
if not isinstance(frame, dict) or frame.get("frame_name") != frame_name:
continue
detections = frame.get("detections")
if not isinstance(detections, list):
raise RuntimeError("reference frame detections must be a list")
selected = [
item
for item in detections
if isinstance(item, dict) and item.get("label") == label
]
return max(selected, key=lambda item: float(item["score"])) if selected else None
raise RuntimeError(f"reference frame not found: {frame_name}")
def _box_iou(left: list[float], right: list[float]) -> float:
intersection_width = max(0.0, min(left[2], right[2]) - max(left[0], right[0]))
intersection_height = max(0.0, min(left[3], right[3]) - max(left[1], right[1]))
intersection = intersection_width * intersection_height
left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1])
right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1])
union = left_area + right_area - intersection
return intersection / union if union > 0 else 0.0
def _validate_metadata(metadata: dict[str, Any]) -> None:
expected_inputs = [{"name": "input", "datatype": "FP32", "shape": [1, 3, 704, 704]}]
expected_outputs = [
{"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]},
{"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]},
]
if metadata.get("inputs") != expected_inputs or metadata.get("outputs") != expected_outputs:
raise RuntimeError(f"unexpected isolated Triton model metadata: {metadata}")
def _write_overlay(
image: Image.Image, detections: list[dict[str, object]], destination: Path
) -> None:
annotated = image.copy()
draw = ImageDraw.Draw(annotated)
for detection in detections:
score = cast(float, detection["score"])
if score < OVERLAY_THRESHOLD:
continue
box = cast(list[float], detection["bbox_xyxy"])
draw.rectangle(box, outline=(0, 220, 112), width=3)
draw.text(
(box[0] + 3, box[1] + 3),
f"{detection['label']} {score:.2f}",
fill=(255, 255, 255),
)
annotated.save(destination)
def _load_mask(path: Path) -> np.ndarray[Any, Any]:
with Image.open(path) as image:
array = np.asarray(image.convert("L"), dtype=np.uint8)
if array.shape != (600, 800):
raise RuntimeError("valid-FOV mask must be 800x600")
return array > 0
def _gpu_sample() -> dict[str, object]:
completed = subprocess.run(
[
"nvidia-smi",
"--query-gpu=name,memory.total,memory.used,utilization.gpu,temperature.gpu,power.draw",
"--format=csv,noheader,nounits",
],
check=True,
capture_output=True,
text=True,
)
values = [value.strip() for value in completed.stdout.strip().split(",")]
if len(values) != 6:
raise RuntimeError("unexpected nvidia-smi response")
return {
"name": values[0],
"memory_total_mib": float(values[1]),
"memory_used_mib": float(values[2]),
"utilization_gpu_percent": float(values[3]),
"temperature_c": float(values[4]),
"power_w": float(values[5]),
}
def _timing_summary(values: list[float]) -> dict[str, float]:
array = np.asarray(values, dtype=np.float64)
return {
"mean": round(float(array.mean()), 6),
"p50": round(float(np.percentile(array, 50)), 6),
"p95": round(float(np.percentile(array, 95)), 6),
"max": round(float(array.max()), 6),
}
def _milliseconds(started: int, completed: int) -> float:
return round(max(0, completed - started) / 1_000_000.0, 6)
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/seal_m48s_fixed_detector_tournament.py
+242
View File
@@ -0,0 +1,242 @@
#!/usr/bin/env python3
"""Seal the bounded M48S fixed-class detector tournament."""
from __future__ import annotations
import argparse
import hashlib
import json
from collections import Counter
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Final
from k1link.perception.fixed_class_detector_tournament import (
TOURNAMENT_SCHEMA,
CandidateWorkerRun,
canonical_json,
false_authority,
sha256_path,
)
RESULT_PREFIX: Final = "m48s-fixed-detector-tournament-"
THRESHOLDS: Final = (0.25, 0.5)
def main() -> int:
repository = Path(__file__).resolve().parents[2]
runtime = repository / ".runtime/compute-experiments/m48s-semantic-shadow"
parser = argparse.ArgumentParser()
parser.add_argument(
"--profile",
type=Path,
default=repository / "config/perception/fixed-class-detector-tournament-v0.json",
)
parser.add_argument(
"--dfine-result",
type=Path,
default=runtime / "fixed-detector-tournament-worker/dfine-s-worker.json",
)
parser.add_argument(
"--rf-detr-result",
type=Path,
default=runtime / "fixed-detector-tournament-worker/rf-detr-large-worker.json",
)
parser.add_argument(
"--yolox-result",
type=Path,
default=(
runtime
/ "yolox-all-coco-results"
/ (
"m48s-yolox-all-coco-shadow-"
"7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
)
),
)
parser.add_argument(
"--output-root",
type=Path,
default=runtime / "fixed-detector-tournament-results",
)
arguments = parser.parse_args()
profile_path = arguments.profile.resolve(strict=True)
profile = _load_object(profile_path)
if profile.get("schema_version") != "missioncore.fixed-class-detector-tournament-profile/v0":
raise RuntimeError("unexpected fixed-class tournament profile schema")
dfine_path = arguments.dfine_result.resolve(strict=True)
rf_detr_path = arguments.rf_detr_result.resolve(strict=True)
dfine = CandidateWorkerRun.from_path(dfine_path)
rf_detr = CandidateWorkerRun.from_path(rf_detr_path)
yolox_root = arguments.yolox_result.resolve(strict=True)
yolox_manifest_path = yolox_root / "manifest.json"
yolox_frames_path = yolox_root / "frames.jsonl"
yolox_manifest = _load_object(yolox_manifest_path)
yolox_frames = _load_jsonl(yolox_frames_path)
if yolox_manifest.get("result_id") != profile["source"]["baseline_result_id"]:
raise RuntimeError("YOLOX baseline result does not match the tournament profile")
worker_paths = {
dfine.profile_id: dfine_path,
rf_detr.profile_id: rf_detr_path,
}
candidate_summaries = {
dfine.profile_id: _candidate_summary(dfine),
rf_detr.profile_id: _candidate_summary(rf_detr),
}
baseline_summary = {
"profile_id": "yolox-s-raw-kb4-all-coco-shadow/v2",
"provider_id": "triton-yolox-s-raw-kb4-all-coco/v2",
"quality": {
str(threshold): _yolox_quality(yolox_frames, threshold)
for threshold in THRESHOLDS
},
"worker_metrics": yolox_manifest["metrics"],
}
evidence = {
"profile_sha256": sha256_path(profile_path),
"worker_result_sha256": {
profile_id: sha256_path(path) for profile_id, path in sorted(worker_paths.items())
},
"yolox_manifest_sha256": sha256_path(yolox_manifest_path),
"yolox_frames_sha256": sha256_path(yolox_frames_path),
"manual_visual_review": {
"reviewed_frames": [253, 275, 443, 1228],
"rf_detr_frame_253_dog_box_correct": True,
"dfine_frame_253_dog_box_present_at_0_25": False,
"dfine_observed_confusions": [
"dog-as-skateboard",
"scanner-body-as-surfboard",
"duplicate-risk-labels-on-one-object",
],
"rf_detr_observed_advantage": "correct dog and cleaner person/vehicle labeling",
"ground_truth": False,
},
}
decision = {
"finalist_profile_id": rf_detr.profile_id,
"finalist_provider_id": rf_detr.provider_id,
"eliminated_profile_ids": [
"yolox-s-raw-kb4-all-coco-shadow/v2",
dfine.profile_id,
],
"reasons": {
"yolox-s-raw-kb4-all-coco-shadow/v2": (
"visible frame-253 dog missed; retained only as regression baseline"
),
dfine.profile_id: (
"frame-253 dog missed at 0.25 and 0.5; more risk-class confusions; slower qualifier"
),
rf_detr.profile_id: (
"correct frame-253 dog at 0.741; cleaner risk labels; 44.786 FPS PyTorch qualifier"
),
},
"candidate_accepted": False,
"next_gate": (
"RF-DETR-L TensorRT FP16 through isolated Triton, then full recorded "
"concurrent-load replay"
),
}
identity = {
"schema_version": TOURNAMENT_SCHEMA,
"profile_id": profile["profile_id"],
"evidence": evidence,
"baseline": baseline_summary,
"candidates": candidate_summaries,
"decision": decision,
"completed": True,
"accepted": False,
"authority": false_authority(),
}
result_id = RESULT_PREFIX + hashlib.sha256(canonical_json(identity)).hexdigest()
output_root = arguments.output_root.absolute()
output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
destination = output_root / result_id
if destination.exists():
raise RuntimeError("immutable fixed-class tournament result already exists")
destination.mkdir(mode=0o700)
for source, name in (
(dfine_path, "dfine-s-worker.json"),
(rf_detr_path, "rf-detr-large-worker.json"),
):
(destination / name).write_bytes(source.read_bytes())
manifest = {"result_id": result_id, **identity}
(destination / "manifest.json").write_bytes(canonical_json(manifest) + b"\n")
(destination / "report.json").write_bytes(
canonical_json(
{
"schema_version": TOURNAMENT_SCHEMA,
"result_id": result_id,
"completed": True,
"accepted": False,
"baseline": baseline_summary,
"candidates": candidate_summaries,
"decision": decision,
"authority": false_authority(),
}
)
+ b"\n"
)
print(result_id)
print(json.dumps(decision, indent=2, sort_keys=True))
return 0
def _candidate_summary(run: CandidateWorkerRun) -> dict[str, object]:
return {
"profile_id": run.profile_id,
"provider_id": run.provider_id,
"upstream_revision": run.upstream_revision,
"checkpoint_sha256": run.checkpoint_sha256,
"quality": {
str(threshold): run.quality_summary(threshold=threshold)
for threshold in THRESHOLDS
},
"worker_metrics": run.metrics,
}
def _yolox_quality(frames: list[Mapping[str, Any]], threshold: float) -> dict[str, object]:
selected = []
dog_selected = []
for frame in frames:
detections = frame.get("all_coco_detections")
if not isinstance(detections, list):
raise RuntimeError("YOLOX frame lacks all-COCO detections")
for detection in detections:
if not isinstance(detection, dict):
raise RuntimeError("YOLOX detection must be an object")
score = detection.get("score")
label = detection.get("label")
if isinstance(score, int | float) and score >= threshold and isinstance(label, str):
selected.append(label)
if frame.get("frame_name") == "frame-000253.jpg" and label == "dog":
dog_selected.append(float(score))
counts = Counter(selected)
return {
"threshold": threshold,
"detection_count": len(selected),
"class_counts": dict(sorted(counts.items())),
"frame_000253_dog_detected": bool(dog_selected),
"frame_000253_dog_max_score": max(dog_selected) if dog_selected else None,
}
def _load_object(path: Path) -> Mapping[str, Any]:
document = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(document, dict):
raise RuntimeError(f"JSON document must be an object: {path}")
return document
def _load_jsonl(path: Path) -> list[Mapping[str, Any]]:
rows: list[Mapping[str, Any]] = []
for line in path.read_text(encoding="utf-8").splitlines():
document = json.loads(line)
if not isinstance(document, dict):
raise RuntimeError(f"JSONL row must be an object: {path}")
rows.append(document)
return rows
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/seal_m48s_rf_detr_deployment.py
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env python3
"""Seal the RF-DETR TensorRT/Triton detector deployment gate."""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any, Final
from k1link.perception.fixed_class_detector_tournament import (
CandidateWorkerRun,
canonical_json,
false_authority,
sha256_path,
)
SCHEMA_VERSION: Final = "missioncore.m48s-rf-detr-deployment-gate/v0"
RESULT_PREFIX: Final = "m48s-rf-detr-deployment-gate-"
ENGINE_SHA256: Final = "986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8"
EXPORTED_ONNX_SHA256: Final = (
"9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300"
)
FP16_ONNX_SHA256: Final = "9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb"
def main() -> int:
repository = Path(__file__).resolve().parents[2]
runtime = repository / ".runtime/compute-experiments/m48s-semantic-shadow"
parser = argparse.ArgumentParser()
parser.add_argument(
"--profile",
type=Path,
default=repository / "config/perception/rf-detr-large-risk-shadow-v0.json",
)
parser.add_argument(
"--tournament",
type=Path,
default=(
runtime
/ "fixed-detector-tournament-results"
/ (
"m48s-fixed-detector-tournament-"
"0e61d75e6dc575d53e4bb98772a41d240fe627ad642de5178beb1154636e1299"
)
/ "manifest.json"
),
)
parser.add_argument(
"--pytorch-result",
type=Path,
default=runtime / "fixed-detector-tournament-worker/rf-detr-large-worker.json",
)
parser.add_argument(
"--triton-result",
type=Path,
default=runtime / "fixed-detector-tournament-worker/rf-detr-large-triton-worker.json",
)
parser.add_argument(
"--onnx-export",
type=Path,
default=runtime / "rf-detr-deployment-worker/rf-detr-large-onnx-export.json",
)
parser.add_argument(
"--fp16-conversion",
type=Path,
default=(
runtime
/ "rf-detr-deployment-worker/rf-detr-large-onnx-fp16-conversion-v4.json"
),
)
parser.add_argument(
"--trtexec-log",
type=Path,
default=runtime / "rf-detr-deployment-worker/rf-detr-large-trtexec-build-v6.log",
)
parser.add_argument(
"--load-result",
type=Path,
default=runtime / "rf-detr-deployment-worker/rf-detr-load-30m.json",
)
parser.add_argument(
"--output-root",
type=Path,
default=runtime / "rf-detr-deployment-results",
)
arguments = parser.parse_args()
paths = {
"profile": arguments.profile.resolve(strict=True),
"tournament": arguments.tournament.resolve(strict=True),
"pytorch_result": arguments.pytorch_result.resolve(strict=True),
"triton_result": arguments.triton_result.resolve(strict=True),
"onnx_export": arguments.onnx_export.resolve(strict=True),
"fp16_conversion": arguments.fp16_conversion.resolve(strict=True),
"trtexec_log": arguments.trtexec_log.resolve(strict=True),
"load_result": arguments.load_result.resolve(strict=True),
}
profile = _load_object(paths["profile"])
tournament = _load_object(paths["tournament"])
pytorch = CandidateWorkerRun.from_path(paths["pytorch_result"])
triton = CandidateWorkerRun.from_path(paths["triton_result"])
triton_document = _load_object(paths["triton_result"])
onnx_export = _load_object(paths["onnx_export"])
fp16_conversion = _load_object(paths["fp16_conversion"])
load_result = _load_object(paths["load_result"])
build_log = paths["trtexec_log"].read_text("utf-8")
_validate(
profile=profile,
tournament=tournament,
pytorch=pytorch,
triton=triton,
triton_document=triton_document,
onnx_export=onnx_export,
fp16_conversion=fp16_conversion,
load_result=load_result,
build_log=build_log,
)
pytorch_quality = pytorch.quality_summary(threshold=0.5)
triton_quality = triton.quality_summary(threshold=0.5)
evidence = {
"files": {
name: {"sha256": sha256_path(path), "size_bytes": path.stat().st_size}
for name, path in sorted(paths.items())
},
"engine_sha256": ENGINE_SHA256,
"exported_onnx_sha256": EXPORTED_ONNX_SHA256,
"strongly_typed_fp16_onnx_sha256": FP16_ONNX_SHA256,
"pytorch_quality_at_0_5": pytorch_quality,
"triton_quality_at_0_5": triton_quality,
"tensorrt_parity": triton_document["metrics"]["pytorch_reference_parity"],
"triton_benchmark": triton_document["metrics"]["benchmark"],
"source_paced_load": {
"execution": load_result["execution"],
"checks": load_result["checks"],
"end_to_end_ms": load_result["metrics"]["end_to_end_ms"],
"detector_completion_age_ms": load_result["metrics"][
"detector_completion_age_ms"
],
"gpu": load_result["metrics"]["gpu"],
},
}
decision = {
"tournament_finalist": True,
"tensorrt_numeric_parity_passed": True,
"detector_source_paced_load_gate_passed": True,
"ready_for_reference_graph_shadow": True,
"integrated_world_state_gate_evaluated": False,
"production_accepted": False,
"next_gate": (
"run the RF-DETR shadow provider inside the complete reference graph and require "
"world-state p95 <= 175 ms without changing false authority"
),
}
identity = {
"schema_version": SCHEMA_VERSION,
"profile_id": profile["profile_id"],
"evidence": evidence,
"decision": decision,
"completed": True,
"accepted": False,
"authority": false_authority(),
}
result_id = RESULT_PREFIX + hashlib.sha256(canonical_json(identity)).hexdigest()
destination = arguments.output_root.absolute() / result_id
if destination.exists():
raise RuntimeError("immutable RF-DETR deployment result already exists")
destination.mkdir(mode=0o700, parents=True)
for name, path in paths.items():
suffix = path.suffix or ".evidence"
(destination / f"{name}{suffix}").write_bytes(path.read_bytes())
manifest = {"result_id": result_id, **identity}
(destination / "manifest.json").write_bytes(canonical_json(manifest) + b"\n")
(destination / "report.json").write_bytes(
canonical_json(
{
"schema_version": SCHEMA_VERSION,
"result_id": result_id,
"completed": True,
"accepted": False,
"evidence": evidence,
"decision": decision,
"authority": false_authority(),
}
)
+ b"\n"
)
print(result_id)
print(json.dumps(decision, indent=2, sort_keys=True))
return 0
def _validate(
*,
profile: dict[str, Any],
tournament: dict[str, Any],
pytorch: CandidateWorkerRun,
triton: CandidateWorkerRun,
triton_document: dict[str, Any],
onnx_export: dict[str, Any],
fp16_conversion: dict[str, Any],
load_result: dict[str, Any],
build_log: str,
) -> None:
if profile.get("schema_version") != "missioncore.rf-detr-risk-shadow-profile/v0":
raise RuntimeError("unexpected RF-DETR shadow profile schema")
decision = tournament.get("decision")
if not isinstance(decision, dict) or decision.get("finalist_profile_id") != pytorch.profile_id:
raise RuntimeError("tournament does not select the RF-DETR PyTorch reference")
if triton.profile_id != "rf-detr-large-coco-704-trt11-fp16/v0":
raise RuntimeError("unexpected RF-DETR Triton profile")
if triton_document.get("engine_sha256") != ENGINE_SHA256:
raise RuntimeError("RF-DETR Triton engine identity changed")
parity = triton_document.get("metrics", {}).get("pytorch_reference_parity", {})
if not isinstance(parity, dict) or parity.get("passed") is not True:
raise RuntimeError("RF-DETR TensorRT numeric parity failed")
if pytorch.quality_summary(threshold=0.5)["class_counts"] != triton.quality_summary(
threshold=0.5
)["class_counts"]:
raise RuntimeError("RF-DETR TensorRT 0.5 class counts diverged from PyTorch")
if onnx_export.get("onnx", {}).get("sha256") != EXPORTED_ONNX_SHA256:
raise RuntimeError("RF-DETR exported ONNX identity changed")
if fp16_conversion.get("output_onnx_sha256") != FP16_ONNX_SHA256:
raise RuntimeError("RF-DETR strongly typed FP16 ONNX identity changed")
required_build_markers = (
"Precision: Strongly Typed",
"Input binding for input with dimensions 1x3x704x704 and type fp32",
"Output binding for dets with dimensions 1x300x4 and type fp16",
"Output binding for labels with dimensions 1x300x91 and type fp16",
"&&&& PASSED TensorRT.trtexec",
)
if any(marker not in build_log for marker in required_build_markers):
raise RuntimeError("TensorRT build log is incomplete")
if (
load_result.get("schema_version")
!= "missioncore.m48s-rf-detr-source-paced-load/v0"
or load_result.get("completed") is not True
or load_result.get("detector_load_gate_passed") is not True
or load_result.get("candidate_accepted") is not False
or load_result.get("integrated_world_state_gate_evaluated") is not False
):
raise RuntimeError("RF-DETR source-paced load result is incompatible")
checks = load_result.get("checks")
if (
not isinstance(checks, dict)
or not checks
or not all(value is True for value in checks.values())
):
raise RuntimeError("RF-DETR source-paced load checks did not all pass")
if load_result.get("authority") != false_authority():
raise RuntimeError("RF-DETR source-paced load gained authority")
def _load_object(path: Path) -> dict[str, Any]:
document = json.loads(path.read_text("utf-8"))
if not isinstance(document, dict):
raise RuntimeError(f"JSON document must be an object: {path}")
return document
if __name__ == "__main__":
raise SystemExit(main())
experiments/perception/worker/Dockerfile.fixed-detector-tournament
+30
View File
@@ -0,0 +1,30 @@
FROM nvcr.io/nvidia/tritonserver:26.06-py3
ARG DFINE_REVISION=956d1709314c2c6a4df6f34de232054578a7449f
RUN python3 -m pip install --no-cache-dir \
--index-url https://download.pytorch.org/whl/cu130 \
"torch==2.9.1+cu130" \
"torchvision==0.24.1+cu130"
RUN git clone https://github.com/Peterande/D-FINE.git /opt/dfine \
&& git -C /opt/dfine checkout --detach "${DFINE_REVISION}" \
&& test "$(git -C /opt/dfine rev-parse HEAD)" = "${DFINE_REVISION}"
RUN python3 -m pip install --no-cache-dir \
"rfdetr[onnx]==1.9.4" \
"numpy==1.26.4" \
"ml_dtypes==0.5.4" \
"onnxconverter-common==1.16.0" \
"tritonclient[http]==2.71.0" \
"faster-coco-eval>=1.6.6" \
"PyYAML>=6.0" \
"scipy>=1.10" \
"calflops>=0.3" \
"loguru>=0.7" \
"tensorboard>=2.17"
LABEL com.nodedc.product="mission-core" \
com.nodedc.stack="ndc-mission-core-compute" \
com.nodedc.role="bounded-detector-qualification" \
com.nodedc.managed-by="codex-bounded-experiment"
experiments/perception/worker/bootstrap_fixed_detector_tournament.sh
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -euo pipefail
readonly DFINE_REVISION="956d1709314c2c6a4df6f34de232054578a7449f"
python3 -m pip install --no-cache-dir \
--index-url https://download.pytorch.org/whl/cu130 \
"torch==2.9.1+cu130" \
"torchvision==0.24.1+cu130"
git clone https://github.com/Peterande/D-FINE.git /opt/dfine
git -C /opt/dfine checkout --detach "${DFINE_REVISION}"
test "$(git -C /opt/dfine rev-parse HEAD)" = "${DFINE_REVISION}"
python3 -m pip install --no-cache-dir \
"rfdetr[onnx]==1.9.4" \
"numpy==1.26.4" \
"ml_dtypes==0.5.4" \
"onnxconverter-common==1.16.0" \
"tritonclient[http]==2.71.0" \
"faster-coco-eval>=1.6.6" \
"PyYAML>=6.0" \
"scipy>=1.10" \
"calflops>=0.3" \
"loguru>=0.7" \
"tensorboard>=2.17"
python3 -c "import importlib.metadata, rfdetr, torch, torchvision; print(torch.__version__, torchvision.__version__, importlib.metadata.version('rfdetr'))"
experiments/perception/worker/build_m48s_rf_detr_tensorrt.sh
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -ne 3 ]]; then
echo "usage: $0 ONNX_PATH ENGINE_PATH LOG_PATH" >&2
exit 2
fi
readonly onnx_path="$1"
readonly engine_path="$2"
readonly log_path="$3"
readonly timing_cache="${engine_path}.timing-cache"
test -f "${onnx_path}"
test ! -e "${engine_path}"
test ! -e "${log_path}"
mkdir -p "$(dirname "${engine_path}")" "$(dirname "${log_path}")"
/usr/bin/trtexec \
--onnx="${onnx_path}" \
--saveEngine="${engine_path}" \
--timingCacheFile="${timing_cache}" \
--memPoolSize=workspace:4096 \
--warmUp=1000 \
--duration=5 \
--avgRuns=100 \
2>&1 | tee "${log_path}"
test -s "${engine_path}"
sha256sum "${onnx_path}" "${engine_path}" "${log_path}"
experiments/perception/worker/rf_detr_large_config.pbtxt
+47
View File
@@ -0,0 +1,47 @@
name: "rf_detr_large"
platform: "tensorrt_plan"
max_batch_size: 0
input [
{
name: "input"
data_type: TYPE_FP32
dims: [ 1, 3, 704, 704 ]
}
]
output [
{
name: "dets"
data_type: TYPE_FP16
dims: [ 1, 300, 4 ]
},
{
name: "labels"
data_type: TYPE_FP16
dims: [ 1, 300, 91 ]
}
]
instance_group [
{
count: 1
kind: KIND_GPU
gpus: [ 0 ]
}
]
model_warmup [
{
name: "rf_detr_large_zero"
batch_size: 0
inputs: {
key: "input"
value: {
data_type: TYPE_FP32
dims: [ 1, 3, 704, 704 ]
zero_data: true
}
}
}
]
src/k1link/perception/detector.py
+154 -4
View File
@@ -1,4 +1,4 @@
"""Frozen raw-KB4 YOLOX provider for class-agnostic object proposals."""
"""Versioned fixed-class detector providers for raw-KB4 object proposals."""
from __future__ import annotations
@@ -14,10 +14,22 @@ from numpy.typing import NDArray
from .contracts import BoundingRegion2D, ObjectProposal2D
from .providers import SourcePacket
from .rf_detr_object_detector import (
RF_DETR_CONFIG,
RF_DETR_MODEL_ID,
RF_DETR_MODEL_VERSION,
RfDetrConfig,
RfDetrDetection,
RfDetrInferenceBackend,
postprocess_rf_detr,
preprocess_raw_kb4_rf_detr,
)
from .yolox_object_detector import (
ALL_COCO_YOLOX_CONFIG,
FROZEN_YOLOX_CONFIG,
YOLOX_MODEL_ID,
YOLOX_MODEL_VERSION,
AllCocoYoloxConfig,
FrozenYoloxConfig,
ImageResizer,
InferenceBackend,
@@ -27,8 +39,12 @@ from .yolox_object_detector import (
)
FROZEN_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4/v1"
ALL_COCO_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4-all-coco/v2"
FROZEN_YOLOX_MODEL_ID: Final = f"{YOLOX_MODEL_ID}:{YOLOX_MODEL_VERSION}"
FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}"
RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0"
class DetectorProviderError(RuntimeError):
@@ -57,7 +73,7 @@ class FrozenYoloxDetectorProvider:
mask: NDArray[np.bool_],
backend: InferenceBackend,
resizer: ImageResizer | None = None,
config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG,
config: FrozenYoloxConfig | AllCocoYoloxConfig = FROZEN_YOLOX_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
@@ -95,7 +111,11 @@ class FrozenYoloxDetectorProvider:
)
output = self.backend.infer(tensor)
postprocessed = postprocess_yolox(output, self.mask, config=self.config)
proposals = proposals_from_detections(packet, postprocessed.detections)
proposals = proposals_from_detections(
packet,
postprocessed.detections,
provider_id=self.provider_id,
)
except Exception:
with self._lock:
self._failed_frames += 1
@@ -125,6 +145,8 @@ class FrozenYoloxDetectorProvider:
def proposals_from_detections(
packet: SourcePacket,
detections: tuple[YoloxDetection, ...],
*,
provider_id: str = FROZEN_YOLOX_PROVIDER_ID,
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
@@ -134,7 +156,7 @@ def proposals_from_detections(
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=FROZEN_YOLOX_PROVIDER_ID,
provider_id=provider_id,
model_id=FROZEN_YOLOX_MODEL_ID,
preprocess_id=FROZEN_YOLOX_PREPROCESS_ID,
semantic_hint=detection.label,
@@ -144,12 +166,140 @@ def proposals_from_detections(
)
class AllCocoYoloxDetectorProvider(FrozenYoloxDetectorProvider):
"""Emit every qualified COCO class without adding another inference pass."""
provider_id: str = ALL_COCO_YOLOX_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: InferenceBackend,
resizer: ImageResizer | None = None,
config: AllCocoYoloxConfig = ALL_COCO_YOLOX_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
super().__init__(
mask=mask,
backend=backend,
resizer=resizer,
config=config,
clock_ns=clock_ns,
)
class RfDetrShadowDetectorProvider:
"""Emit behavior-relevant fixed classes from one RF-DETR inference pass."""
provider_id: str = RF_DETR_SHADOW_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: RfDetrInferenceBackend,
resizer: ImageResizer | None = None,
config: RfDetrConfig = RF_DETR_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
raise DetectorProviderError("RF-DETR valid-FOV mask is incompatible")
self.mask = np.asarray(mask, dtype=np.bool_)
self.backend = backend
self.resizer = resizer
self.config = config
self._clock_ns = clock_ns
self._lock = Lock()
self._input_frames = 0
self._completed_frames = 0
self._failed_frames = 0
self._zero_proposal_frames = 0
self._proposal_count = 0
self._rejected: Counter[str] = Counter()
self._core_duration_ns = 0
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
payload = packet.image_payload
with self._lock:
self._input_frames += 1
started_ns = int(self._clock_ns())
try:
if not isinstance(payload, np.ndarray):
raise DetectorProviderError("RF-DETR requires a decoded BGR image payload")
image = np.asarray(payload)
if image.dtype != np.uint8:
raise DetectorProviderError("decoded BGR image must be uint8")
tensor = preprocess_raw_kb4_rf_detr(
image,
self.mask,
config=self.config,
resizer=self.resizer,
)
output = self.backend.infer(tensor)
postprocessed = postprocess_rf_detr(output, self.mask, config=self.config)
proposals = proposals_from_rf_detr_detections(packet, postprocessed.detections)
except Exception:
with self._lock:
self._failed_frames += 1
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
raise
with self._lock:
self._completed_frames += 1
self._proposal_count += len(proposals)
self._zero_proposal_frames += not proposals
self._rejected.update(dict(postprocessed.rejected))
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
return proposals
def snapshot(self) -> DetectorProviderSnapshot:
with self._lock:
return DetectorProviderSnapshot(
input_frames=self._input_frames,
completed_frames=self._completed_frames,
failed_frames=self._failed_frames,
zero_proposal_frames=self._zero_proposal_frames,
proposal_count=self._proposal_count,
rejected=tuple(sorted(self._rejected.items())),
core_duration_ns=self._core_duration_ns,
)
def proposals_from_rf_detr_detections(
packet: SourcePacket,
detections: tuple[RfDetrDetection, ...],
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
ObjectProposal2D(
proposal_id=f"proposal-{envelope.sequence}-{index}",
source_id=envelope.source_id,
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=RF_DETR_SHADOW_PROVIDER_ID,
model_id=RF_DETR_SHADOW_MODEL_ID,
preprocess_id=RF_DETR_SHADOW_PREPROCESS_ID,
semantic_hint=detection.label,
provider_tracklet=None,
)
for index, detection in enumerate(detections)
)
__all__ = [
"ALL_COCO_YOLOX_PROVIDER_ID",
"FROZEN_YOLOX_MODEL_ID",
"FROZEN_YOLOX_PREPROCESS_ID",
"FROZEN_YOLOX_PROVIDER_ID",
"RF_DETR_SHADOW_MODEL_ID",
"RF_DETR_SHADOW_PREPROCESS_ID",
"RF_DETR_SHADOW_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"AllCocoYoloxDetectorProvider",
"FrozenYoloxDetectorProvider",
"RfDetrShadowDetectorProvider",
"proposals_from_detections",
"proposals_from_rf_detr_detections",
]
src/k1link/perception/fixed_class_detector_tournament.py
+285
View File
@@ -0,0 +1,285 @@
"""Contracts for the bounded M48S fixed-class detector tournament."""
from __future__ import annotations
import hashlib
import json
from collections import Counter
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Final
WORKER_RUN_SCHEMA: Final = "missioncore.m48s-fixed-detector-candidate-worker/v0"
TOURNAMENT_SCHEMA: Final = "missioncore.m48s-fixed-detector-tournament/v0"
EXACT_FRAME_NAMES: Final = (
"frame-000121.png",
"frame-000131.png",
"frame-000253.png",
"frame-000275.png",
"frame-000443.png",
"frame-000463.png",
"frame-001094.png",
"frame-001228.png",
"frame-001454.png",
"frame-001856.png",
"frame-002386.png",
)
RISK_GROUPS: Final[Mapping[str, frozenset[str]]] = {
"person": frozenset({"person"}),
"animal": frozenset(
{"bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe"}
),
"light-road-user": frozenset({"bicycle", "motorcycle", "skateboard"}),
"vehicle": frozenset({"car", "bus", "truck"}),
}
_RISK_LABELS: Final = frozenset().union(*RISK_GROUPS.values())
_FALSE_AUTHORITY: Final = {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
class FixedClassTournamentError(ValueError):
"""Raised when tournament evidence violates its bounded contract."""
@dataclass(frozen=True)
class CandidateDetection:
"""One normalized COCO detection emitted by a candidate."""
class_id: int
label: str
score: float
bbox_xyxy: tuple[float, float, float, float]
valid_fov_fraction: float
@classmethod
def from_document(cls, document: object) -> CandidateDetection:
item = _mapping(document, "detection")
box = item.get("bbox_xyxy")
if not isinstance(box, list) or len(box) != 4:
raise FixedClassTournamentError("detection bbox_xyxy must contain four numbers")
values = (
_number(box[0], "bbox coordinate"),
_number(box[1], "bbox coordinate"),
_number(box[2], "bbox coordinate"),
_number(box[3], "bbox coordinate"),
)
x1, y1, x2, y2 = values
if x2 <= x1 or y2 <= y1:
raise FixedClassTournamentError("detection box must have positive area")
score = _number(item.get("score"), "detection score")
valid_fov_fraction = _number(
item.get("valid_fov_fraction"), "detection valid-FOV fraction"
)
if not 0.0 <= score <= 1.0:
raise FixedClassTournamentError("detection score must be in [0, 1]")
if not 0.0 <= valid_fov_fraction <= 1.0:
raise FixedClassTournamentError("valid-FOV fraction must be in [0, 1]")
return cls(
class_id=_integer(item.get("class_id"), "detection class id"),
label=_text(item.get("label"), "detection label"),
score=score,
bbox_xyxy=values,
valid_fov_fraction=valid_fov_fraction,
)
@property
def risk_group(self) -> str | None:
for group, labels in RISK_GROUPS.items():
if self.label in labels:
return group
return None
@dataclass(frozen=True)
class CandidateFrame:
"""One exact-frame candidate result."""
frame_name: str
source_sha256: str
detections: tuple[CandidateDetection, ...]
end_to_end_ms: float
@classmethod
def from_document(cls, document: object) -> CandidateFrame:
item = _mapping(document, "frame")
detections = item.get("detections")
if not isinstance(detections, list):
raise FixedClassTournamentError("frame detections must be a list")
return cls(
frame_name=_text(item.get("frame_name"), "frame name"),
source_sha256=_digest(item.get("source_sha256"), "source digest"),
detections=tuple(CandidateDetection.from_document(value) for value in detections),
end_to_end_ms=_nonnegative_number(
_mapping(item.get("timing_ms"), "frame timing").get("end_to_end"),
"frame end-to-end timing",
),
)
@dataclass(frozen=True)
class CandidateWorkerRun:
"""Validated raw Worker result for one candidate."""
profile_id: str
provider_id: str
upstream_revision: str
checkpoint_sha256: str
frames: tuple[CandidateFrame, ...]
metrics: Mapping[str, object]
authority: Mapping[str, bool]
@classmethod
def from_document(cls, document: object) -> CandidateWorkerRun:
root = _mapping(document, "worker result")
if root.get("schema_version") != WORKER_RUN_SCHEMA:
raise FixedClassTournamentError("unexpected candidate Worker schema")
frames_raw = root.get("frames")
if not isinstance(frames_raw, list):
raise FixedClassTournamentError("worker result frames must be a list")
frames = tuple(CandidateFrame.from_document(value) for value in frames_raw)
if tuple(sorted(frame.frame_name for frame in frames)) != EXACT_FRAME_NAMES:
raise FixedClassTournamentError("worker result does not contain the exact M48S slice")
if len({frame.frame_name for frame in frames}) != len(EXACT_FRAME_NAMES):
raise FixedClassTournamentError("worker result contains duplicate frames")
execution = _mapping(root.get("execution"), "worker execution")
if execution.get("inference_passes_per_evidence_frame") != 1:
raise FixedClassTournamentError(
"candidate must use one inference pass per evidence frame"
)
authority = _boolean_mapping(root.get("authority"), "worker authority")
if authority != _FALSE_AUTHORITY:
raise FixedClassTournamentError("candidate Worker result must retain false authority")
completed = root.get("completed")
if completed is not True:
raise FixedClassTournamentError("candidate Worker result is incomplete")
return cls(
profile_id=_text(root.get("profile_id"), "profile id"),
provider_id=_text(root.get("provider_id"), "provider id"),
upstream_revision=_text(root.get("upstream_revision"), "upstream revision"),
checkpoint_sha256=_digest(root.get("checkpoint_sha256"), "checkpoint digest"),
frames=frames,
metrics=_mapping(root.get("metrics"), "worker metrics"),
authority=authority,
)
@classmethod
def from_path(cls, path: Path) -> CandidateWorkerRun:
try:
document = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise FixedClassTournamentError(
f"cannot read candidate Worker result: {path}"
) from error
return cls.from_document(document)
def quality_summary(self, *, threshold: float) -> dict[str, object]:
if not 0.0 <= threshold <= 1.0:
raise FixedClassTournamentError("quality threshold must be in [0, 1]")
selected = tuple(
detection
for frame in self.frames
for detection in frame.detections
if detection.score >= threshold
)
class_counts = Counter(item.label for item in selected)
risk_counts = Counter(item.risk_group for item in selected if item.risk_group is not None)
dog_frame = next(frame for frame in self.frames if frame.frame_name == "frame-000253.png")
dog_detections = tuple(
detection
for detection in dog_frame.detections
if detection.label == "dog" and detection.score >= threshold
)
return {
"threshold": threshold,
"detection_count": len(selected),
"class_counts": dict(sorted(class_counts.items())),
"risk_group_counts": dict(sorted(risk_counts.items())),
"risk_detection_count": sum(1 for item in selected if item.label in _RISK_LABELS),
"frame_000253_dog_detected": bool(dog_detections),
"frame_000253_dog_max_score": (
round(max(item.score for item in dog_detections), 6) if dog_detections else None
),
}
def canonical_json(value: object) -> bytes:
"""Return deterministic JSON bytes for immutable evidence identities."""
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
def sha256_path(path: Path) -> str:
"""Hash a file without loading it into memory."""
digest = hashlib.sha256()
try:
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
except OSError as error:
raise FixedClassTournamentError(f"cannot hash evidence file: {path}") from error
return digest.hexdigest()
def false_authority() -> dict[str, bool]:
"""Return a fresh false-authority document."""
return dict(_FALSE_AUTHORITY)
def _mapping(value: object, name: str) -> Mapping[str, Any]:
if not isinstance(value, dict):
raise FixedClassTournamentError(f"{name} must be an object")
return value
def _boolean_mapping(value: object, name: str) -> Mapping[str, bool]:
mapping = _mapping(value, name)
if set(mapping) != set(_FALSE_AUTHORITY) or not all(
isinstance(item, bool) for item in mapping.values()
):
raise FixedClassTournamentError(f"{name} must contain the exact boolean authority fields")
return mapping
def _text(value: object, name: str) -> str:
if not isinstance(value, str) or not value.strip():
raise FixedClassTournamentError(f"{name} must be non-empty text")
return value
def _digest(value: object, name: str) -> str:
text = _text(value, name)
if len(text) != 64 or any(character not in "0123456789abcdef" for character in text):
raise FixedClassTournamentError(f"{name} must be a lowercase SHA-256 digest")
return text
def _integer(value: object, name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise FixedClassTournamentError(f"{name} must be an integer")
return value
def _number(value: object, name: str) -> float:
if isinstance(value, bool) or not isinstance(value, int | float):
raise FixedClassTournamentError(f"{name} must be numeric")
return float(value)
def _nonnegative_number(value: object, name: str) -> float:
result = _number(value, name)
if result < 0.0:
raise FixedClassTournamentError(f"{name} must be non-negative")
return result
src/k1link/perception/recorded_source.py
+11 -5
View File
@@ -188,20 +188,26 @@ class DecodedRecordedSource:
def packets(self, stop_event: Event) -> Iterator[SourcePacket]:
images = self.decoder.frames(stop_event)
for packet in self.source.packets(stop_event):
try:
image = next(images)
image: NDArray[np.uint8] | None = next(images)
except StopIteration as exc:
if stop_event.is_set():
return
raise RecordedSourceError("decoded image stream is empty") from exc
for packet in self.source.packets(stop_event):
if image is None:
raise RecordedSourceError(
"decoded image stream ended before source timeline"
) from exc
)
if image.shape != (600, 800, 3) or image.dtype != np.uint8:
raise RecordedSourceError("decoded image raster is incompatible")
yield replace(packet, image_payload=image)
if not stop_event.is_set():
try:
next(images)
image = next(images)
except StopIteration:
image = None
if not stop_event.is_set():
if image is None:
return
raise RecordedSourceError("decoded image stream exceeds source timeline")
src/k1link/perception/rf_detr_object_detector.py
+401
View File
@@ -0,0 +1,401 @@
"""Pinned RF-DETR-L TensorRT shadow detector for behavior-relevant COCO classes."""
from __future__ import annotations
import http.client
import json
import math
import urllib.parse
from collections import Counter
from dataclasses import dataclass
from typing import Final, Protocol, cast
import numpy as np
from numpy.typing import NDArray
from .yolox_object_detector import COCO_CLASSES, ImageResizer, OpenCvBilinearResizer
RF_DETR_MODEL_ID: Final = "rf_detr_large"
RF_DETR_MODEL_VERSION: Final = 1
RF_DETR_CHECKPOINT_SHA256: Final = (
"0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38"
)
RF_DETR_ONNX_SHA256: Final = (
"9c1948e56bbb6ff03349012b8bb334cacaf8ae480f22caa0704ee70de9a72300"
)
RF_DETR_FP16_ONNX_SHA256: Final = (
"9015fcc1317f268ce866bed6b5a33132c24963e1502b02f145fa184e11de5ecb"
)
RF_DETR_ENGINE_SHA256: Final = (
"986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8"
)
COCO_SPARSE_IDS: Final = (
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42,
43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60,
61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81,
82, 84, 85, 86, 87, 88, 89, 90,
)
COCO_SPARSE_TO_CONTIGUOUS: Final = {
sparse_id: contiguous_id for contiguous_id, sparse_id in enumerate(COCO_SPARSE_IDS)
}
RISK_CLASS_IDS: Final = (
0, # person
1, # bicycle
2, # car
3, # motorcycle
5, # bus
7, # truck
14, 15, 16, 17, 18, 19, 20, 21, 22, 23, # animals
36, # skateboard / light road user proxy
)
_MEANS: Final = np.asarray((0.485, 0.456, 0.406), dtype=np.float32)
_STDS: Final = np.asarray((0.229, 0.224, 0.225), dtype=np.float32)
class RfDetrDetectorError(RuntimeError):
"""The RF-DETR profile, tensor or inference response is incompatible."""
@dataclass(frozen=True, slots=True)
class RfDetrConfig:
source_width: int = 800
source_height: int = 600
input_width: int = 704
input_height: int = 704
fill_value: int = 114
minimum_score: float = 0.25
target_class_ids: tuple[int, ...] = RISK_CLASS_IDS
maximum_detections: int = 300
minimum_box_area_pixels: float = 64.0
maximum_box_area_fraction: float = 0.5
minimum_valid_fov_fraction: float = 0.5
require_center_inside_valid_fov: bool = True
def __post_init__(self) -> None:
if (
self.source_width,
self.source_height,
self.input_width,
self.input_height,
self.fill_value,
self.minimum_score,
self.target_class_ids,
self.maximum_detections,
self.minimum_box_area_pixels,
self.maximum_box_area_fraction,
self.minimum_valid_fov_fraction,
self.require_center_inside_valid_fov,
) != (
800,
600,
704,
704,
114,
0.25,
RISK_CLASS_IDS,
300,
64.0,
0.5,
0.5,
True,
):
raise RfDetrDetectorError("RF-DETR shadow profile cannot be tuned in place")
RF_DETR_CONFIG: Final = RfDetrConfig()
@dataclass(frozen=True, slots=True)
class RfDetrRawOutput:
boxes: NDArray[np.float16]
logits: NDArray[np.float16]
class RfDetrInferenceBackend(Protocol):
def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput: ...
@dataclass(frozen=True, slots=True)
class RfDetrDetection:
class_id: int
label: str
score: float
bbox_xyxy: tuple[float, float, float, float]
valid_fov_fraction: float
def __post_init__(self) -> None:
if not 0 <= self.class_id < len(COCO_CLASSES):
raise RfDetrDetectorError("RF-DETR class id is invalid")
if self.label != COCO_CLASSES[self.class_id]:
raise RfDetrDetectorError("RF-DETR class label is invalid")
if not math.isfinite(self.score) or not 0.0 <= self.score <= 1.0:
raise RfDetrDetectorError("RF-DETR score is invalid")
x1, y1, x2, y2 = self.bbox_xyxy
if not all(math.isfinite(value) for value in self.bbox_xyxy) or not (
0.0 <= x1 < x2 <= 800.0 and 0.0 <= y1 < y2 <= 600.0
):
raise RfDetrDetectorError("RF-DETR source bounding box is invalid")
if not 0.0 <= self.valid_fov_fraction <= 1.0:
raise RfDetrDetectorError("RF-DETR valid-FOV fraction is invalid")
@dataclass(frozen=True, slots=True)
class RfDetrPostprocessResult:
detections: tuple[RfDetrDetection, ...]
rejected: tuple[tuple[str, int], ...]
class TritonRfDetrHttpInferenceBackend:
"""Persistent Triton V2 HTTP transport for the strongly typed FP16 engine."""
def __init__(self, endpoint: str, *, timeout_seconds: float = 60.0) -> None:
parsed = urllib.parse.urlsplit(endpoint)
if (
parsed.scheme != "http"
or not parsed.hostname
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
):
raise RfDetrDetectorError("Triton endpoint must be an explicit HTTP origin")
if not math.isfinite(timeout_seconds) or timeout_seconds <= 0:
raise RfDetrDetectorError("Triton timeout must be positive")
self.path = (
f"{parsed.path.rstrip('/')}/v2/models/{RF_DETR_MODEL_ID}"
f"/versions/{RF_DETR_MODEL_VERSION}/infer"
)
self.connection = http.client.HTTPConnection(
parsed.hostname,
parsed.port or 80,
timeout=timeout_seconds,
)
def close(self) -> None:
self.connection.close()
def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput:
contiguous = np.ascontiguousarray(tensor, dtype=np.float32)
if contiguous.shape != (1, 3, 704, 704) or not np.isfinite(contiguous).all():
raise RfDetrDetectorError("Triton RF-DETR input tensor is incompatible")
binary = contiguous.tobytes()
header = {
"inputs": [
{
"name": "input",
"shape": [1, 3, 704, 704],
"datatype": "FP32",
"parameters": {"binary_data_size": len(binary)},
}
],
"outputs": [
{"name": "dets", "parameters": {"binary_data": True}},
{"name": "labels", "parameters": {"binary_data": True}},
],
}
encoded = json.dumps(header, sort_keys=True, separators=(",", ":")).encode()
self.connection.request(
"POST",
self.path,
body=encoded + binary,
headers={
"Content-Type": "application/octet-stream",
"Inference-Header-Content-Length": str(len(encoded)),
},
)
response = self.connection.getresponse()
payload = response.read()
if response.status != 200:
raise RfDetrDetectorError(
f"Triton RF-DETR inference failed with HTTP {response.status}"
)
header_value = response.getheader("Inference-Header-Content-Length")
try:
header_length = int(header_value or "")
descriptor = json.loads(payload[:header_length])
outputs = descriptor["outputs"]
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
raise RfDetrDetectorError("Triton RF-DETR output descriptor is invalid") from exc
if not isinstance(outputs, list) or len(outputs) != 2:
raise RfDetrDetectorError("Triton RF-DETR output count changed")
offset = header_length
arrays: dict[str, NDArray[np.float16]] = {}
for output, expected_name, expected_shape in zip(
outputs,
("dets", "labels"),
((1, 300, 4), (1, 300, 91)),
strict=True,
):
try:
name = output["name"]
datatype = output["datatype"]
shape = tuple(int(value) for value in output["shape"])
byte_length = int(output["parameters"]["binary_data_size"])
except (KeyError, TypeError, ValueError) as exc:
raise RfDetrDetectorError(
"Triton RF-DETR output descriptor is incomplete"
) from exc
expected_bytes = math.prod(expected_shape) * np.dtype("<f2").itemsize
if (
name != expected_name
or datatype != "FP16"
or shape != expected_shape
or byte_length != expected_bytes
or offset + byte_length > len(payload)
):
raise RfDetrDetectorError("Triton RF-DETR output identity changed")
array = np.frombuffer(payload[offset : offset + byte_length], dtype="<f2")
arrays[name] = np.asarray(array.reshape(shape), dtype=np.float16)
offset += byte_length
if offset != len(payload):
raise RfDetrDetectorError("Triton RF-DETR output byte length changed")
return RfDetrRawOutput(boxes=arrays["dets"], logits=arrays["labels"])
def preprocess_raw_kb4_rf_detr(
image_bgr: NDArray[np.uint8],
mask: NDArray[np.bool_],
*,
config: RfDetrConfig = RF_DETR_CONFIG,
resizer: ImageResizer | None = None,
) -> NDArray[np.float32]:
if image_bgr.shape != (config.source_height, config.source_width, 3):
raise RfDetrDetectorError("raw KB4 image raster changed")
if image_bgr.dtype != np.uint8 or mask.shape != image_bgr.shape[:2] or mask.dtype != np.bool_:
raise RfDetrDetectorError("raw KB4 image or valid-FOV mask type changed")
masked_bgr = np.where(mask[..., None], image_bgr, config.fill_value).astype(np.uint8)
rgb = np.ascontiguousarray(masked_bgr[:, :, ::-1])
resized = (resizer or OpenCvBilinearResizer()).resize(
rgb,
config.input_width,
config.input_height,
)
if resized.shape != (config.input_height, config.input_width, 3):
raise RfDetrDetectorError("resize backend returned an incompatible raster")
normalized = resized.astype(np.float32) / 255.0
normalized = (normalized - _MEANS) / _STDS
return np.ascontiguousarray(normalized.transpose(2, 0, 1), dtype=np.float32)[None]
def postprocess_rf_detr(
output: RfDetrRawOutput,
mask: NDArray[np.bool_],
*,
config: RfDetrConfig = RF_DETR_CONFIG,
) -> RfDetrPostprocessResult:
if output.boxes.shape != (1, 300, 4) or output.logits.shape != (1, 300, 91):
raise RfDetrDetectorError("RF-DETR output tensor shapes are incompatible")
if output.boxes.dtype != np.float16 or output.logits.dtype != np.float16:
raise RfDetrDetectorError("RF-DETR output tensor types are incompatible")
if not np.isfinite(output.boxes).all() or not np.isfinite(output.logits).all():
raise RfDetrDetectorError("RF-DETR output contains non-finite values")
if mask.shape != (config.source_height, config.source_width) or mask.dtype != np.bool_:
raise RfDetrDetectorError("valid-FOV mask is incompatible")
logits = output.logits[0].astype(np.float32)
probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits, -80.0, 80.0)))
flattened = probabilities.reshape(-1)
topk = np.argsort(-flattened, kind="stable")[: config.maximum_detections]
integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1)
rejected: Counter[str] = Counter()
result: list[RfDetrDetection] = []
for flat_index in topk:
score = float(flattened[flat_index])
if score <= config.minimum_score:
continue
query_index = int(flat_index // output.logits.shape[2])
sparse_class_id = int(flat_index % output.logits.shape[2])
class_id = COCO_SPARSE_TO_CONTIGUOUS.get(sparse_class_id)
if class_id is None:
rejected["unmapped-class-slot"] += 1
continue
if class_id not in config.target_class_ids:
rejected["non-risk-class"] += 1
continue
center_x, center_y, box_width, box_height = (
float(value) for value in output.boxes[0, query_index].astype(np.float32)
)
box = np.asarray(
(
(center_x - box_width / 2.0) * config.source_width,
(center_y - box_height / 2.0) * config.source_height,
(center_x + box_width / 2.0) * config.source_width,
(center_y + box_height / 2.0) * config.source_height,
),
dtype=np.float32,
)
box[[0, 2]] = np.clip(box[[0, 2]], 0, config.source_width)
box[[1, 3]] = np.clip(box[[1, 3]], 0, config.source_height)
fraction, center_inside, area = _valid_fraction(box, integral)
if area < config.minimum_box_area_pixels:
rejected["small-box"] += 1
continue
if area / (config.source_width * config.source_height) > config.maximum_box_area_fraction:
rejected["large-box"] += 1
continue
if fraction < config.minimum_valid_fov_fraction:
rejected["outside-valid-fov"] += 1
continue
if config.require_center_inside_valid_fov and not center_inside:
rejected["center-outside-valid-fov"] += 1
continue
result.append(
RfDetrDetection(
class_id=class_id,
label=COCO_CLASSES[class_id],
score=round(score, 9),
bbox_xyxy=cast(
tuple[float, float, float, float],
tuple(round(float(value), 6) for value in box),
),
valid_fov_fraction=round(fraction, 6),
)
)
result.sort(key=lambda item: (-item.score, item.class_id))
return RfDetrPostprocessResult(tuple(result), tuple(sorted(rejected.items())))
def _valid_fraction(
box: NDArray[np.float32], integral: NDArray[np.int64]
) -> tuple[float, bool, float]:
height = integral.shape[0] - 1
width = integral.shape[1] - 1
x1 = int(np.clip(math.floor(float(box[0])), 0, width))
y1 = int(np.clip(math.floor(float(box[1])), 0, height))
x2 = int(np.clip(math.ceil(float(box[2])), 0, width))
y2 = int(np.clip(math.ceil(float(box[3])), 0, height))
area = float(max(0, x2 - x1) * max(0, y2 - y1))
if area <= 0:
return 0.0, False, 0.0
inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1]
center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1))
center_y = int(np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1))
center_inside = bool(
integral[center_y + 1, center_x + 1]
- integral[center_y, center_x + 1]
- integral[center_y + 1, center_x]
+ integral[center_y, center_x]
)
return float(inside) / area, center_inside, area
__all__ = [
"COCO_SPARSE_IDS",
"RF_DETR_CHECKPOINT_SHA256",
"RF_DETR_CONFIG",
"RF_DETR_ENGINE_SHA256",
"RF_DETR_FP16_ONNX_SHA256",
"RF_DETR_MODEL_ID",
"RF_DETR_MODEL_VERSION",
"RF_DETR_ONNX_SHA256",
"RISK_CLASS_IDS",
"RfDetrConfig",
"RfDetrDetection",
"RfDetrDetectorError",
"RfDetrInferenceBackend",
"RfDetrPostprocessResult",
"RfDetrRawOutput",
"TritonRfDetrHttpInferenceBackend",
"postprocess_rf_detr",
"preprocess_raw_kb4_rf_detr",
]
src/k1link/perception/yolox_object_detector.py
+57 -3
View File
@@ -43,6 +43,7 @@ COCO_CLASSES: Final = (
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
"scissors", "teddy bear", "hair drier", "toothbrush",
)
ALL_COCO_CLASS_IDS: Final = tuple(range(len(COCO_CLASSES)))
class YoloxDetectorError(RuntimeError):
@@ -93,6 +94,58 @@ class FrozenYoloxConfig:
FROZEN_YOLOX_CONFIG: Final = FrozenYoloxConfig()
@dataclass(frozen=True, slots=True)
class AllCocoYoloxConfig:
"""Versioned all-COCO shadow profile using the exact frozen YOLOX tensor."""
source_width: int = 800
source_height: int = 600
input_width: int = 640
input_height: int = 640
fill_value: int = 114
minimum_score: float = 0.5
nms_iou_threshold: float = 0.45
target_class_ids: tuple[int, ...] = ALL_COCO_CLASS_IDS
minimum_box_area_pixels: float = 64.0
maximum_box_area_fraction: float = 0.5
minimum_valid_fov_fraction: float = 0.5
require_center_inside_valid_fov: bool = True
def __post_init__(self) -> None:
if (
self.source_width,
self.source_height,
self.input_width,
self.input_height,
self.fill_value,
self.minimum_score,
self.nms_iou_threshold,
self.target_class_ids,
self.minimum_box_area_pixels,
self.maximum_box_area_fraction,
self.minimum_valid_fov_fraction,
self.require_center_inside_valid_fov,
) != (
800,
600,
640,
640,
114,
0.5,
0.45,
ALL_COCO_CLASS_IDS,
64.0,
0.5,
0.5,
True,
):
raise YoloxDetectorError("all-COCO YOLOX profile cannot be tuned in place")
ALL_COCO_YOLOX_CONFIG: Final = AllCocoYoloxConfig()
type YoloxPostprocessConfig = FrozenYoloxConfig | AllCocoYoloxConfig
@dataclass(frozen=True, slots=True)
class YoloxDetection:
class_id: int
@@ -231,7 +284,7 @@ def preprocess_raw_kb4(
image_bgr: NDArray[np.uint8],
mask: NDArray[np.bool_],
*,
config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG,
config: YoloxPostprocessConfig = FROZEN_YOLOX_CONFIG,
resizer: ImageResizer | None = None,
) -> NDArray[np.float32]:
if image_bgr.shape != (config.source_height, config.source_width, 3):
@@ -261,7 +314,7 @@ def postprocess_yolox(
output: NDArray[np.float32],
mask: NDArray[np.bool_],
*,
config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG,
config: YoloxPostprocessConfig = FROZEN_YOLOX_CONFIG,
) -> YoloxPostprocessResult:
if output.shape != (1, 8400, 85) or not np.isfinite(output).all():
raise YoloxDetectorError("YOLOX output tensor is incompatible")
@@ -420,9 +473,10 @@ def _sha256(path: Path) -> str:
__all__ = [
"ALL_COCO_CLASS_IDS", "ALL_COCO_YOLOX_CONFIG", "COCO_CLASSES",
"YOLOX_CONFIG_SHA256", "YOLOX_MODEL_ID", "YOLOX_MODEL_SHA256",
"YOLOX_MODEL_VERSION", "YOLOX_VALID_FOV_SHA256", "FROZEN_YOLOX_CONFIG",
"FrozenYoloxConfig",
"AllCocoYoloxConfig", "FrozenYoloxConfig", "YoloxPostprocessConfig",
"ImageResizer", "InferenceBackend", "OpenCvBilinearResizer",
"TritonHttpInferenceBackend", "YoloxDetection", "YoloxDetectorError",
"YoloxPostprocessResult", "load_valid_fov_mask", "postprocess_yolox",
tests/test_fixed_class_detector_tournament.py
+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
from copy import deepcopy
import pytest
from k1link.perception.fixed_class_detector_tournament import (
EXACT_FRAME_NAMES,
WORKER_RUN_SCHEMA,
CandidateWorkerRun,
FixedClassTournamentError,
false_authority,
)
def _worker_document() -> dict[str, object]:
frames = []
for name in EXACT_FRAME_NAMES:
detections = []
if name == "frame-000253.png":
detections = [
{
"class_id": 16,
"label": "dog",
"score": 0.72,
"bbox_xyxy": [100.0, 200.0, 160.0, 280.0],
"valid_fov_fraction": 1.0,
}
]
frames.append(
{
"frame_name": name,
"source_sha256": "a" * 64,
"detections": detections,
"timing_ms": {"end_to_end": 12.0},
}
)
return {
"schema_version": WORKER_RUN_SCHEMA,
"profile_id": "candidate/v0",
"provider_id": "shadow-candidate/v0",
"upstream_revision": "revision",
"checkpoint_sha256": "b" * 64,
"completed": True,
"execution": {"inference_passes_per_evidence_frame": 1},
"frames": frames,
"metrics": {"capacity_fps": 80.0},
"authority": false_authority(),
}
def test_candidate_worker_run_reports_risk_only_quality() -> None:
result = CandidateWorkerRun.from_document(_worker_document())
summary = result.quality_summary(threshold=0.5)
assert summary["detection_count"] == 1
assert summary["risk_group_counts"] == {"animal": 1}
assert summary["frame_000253_dog_detected"] is True
assert summary["frame_000253_dog_max_score"] == 0.72
def test_candidate_worker_run_rejects_missing_frame() -> None:
document = _worker_document()
frames = document["frames"]
assert isinstance(frames, list)
frames.pop()
with pytest.raises(FixedClassTournamentError, match="exact M48S slice"):
CandidateWorkerRun.from_document(document)
def test_candidate_worker_run_rejects_authority() -> None:
document = deepcopy(_worker_document())
authority = document["authority"]
assert isinstance(authority, dict)
authority["candidate_accepted"] = True
with pytest.raises(FixedClassTournamentError, match="false authority"):
CandidateWorkerRun.from_document(document)
tests/test_m48s_rf_detr_deployment_result.py
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
import json
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
RESULT_ID = (
"m48s-rf-detr-deployment-gate-"
"2feb9e1b12a5588951ad35d63bf23cf6bdd579d54b5329d46d7696f88c444547"
)
RESULT_ROOT = (
REPOSITORY_ROOT
/ ".runtime/compute-experiments/m48s-semantic-shadow/rf-detr-deployment-results"
/ RESULT_ID
)
def test_rf_detr_tensorrt_detector_is_ready_only_for_reference_graph_shadow() -> None:
manifest = json.loads((RESULT_ROOT / "manifest.json").read_text("utf-8"))
decision = manifest["decision"]
evidence = manifest["evidence"]
load = evidence["source_paced_load"]
assert manifest["result_id"] == RESULT_ID
assert manifest["completed"] is True
assert manifest["accepted"] is False
assert decision == {
"detector_source_paced_load_gate_passed": True,
"integrated_world_state_gate_evaluated": False,
"next_gate": (
"run the RF-DETR shadow provider inside the complete reference graph and "
"require world-state p95 <= 175 ms without changing false authority"
),
"production_accepted": False,
"ready_for_reference_graph_shadow": True,
"tensorrt_numeric_parity_passed": True,
"tournament_finalist": True,
}
assert evidence["engine_sha256"] == (
"986399ce706b7380472cf5e473232249fed6e628971d8007f6609e83128d46b8"
)
assert evidence["tensorrt_parity"]["passed"] is True
assert evidence["tensorrt_parity"]["frame_000253_dog_present_in_tensorrt"] is True
assert evidence["pytorch_quality_at_0_5"]["class_counts"] == evidence[
"triton_quality_at_0_5"
]["class_counts"]
assert load["execution"]["source_frames_consumed"] == 18_008
assert load["execution"]["source_frame_replacements"] == 0
assert load["execution"]["effective_consumed_fps"] >= 9.5
assert load["detector_completion_age_ms"]["p95"] <= 175.0
assert load["gpu"]["gpu_memory_used_mib"]["maximum"] <= 20 * 1024
assert load["gpu"]["longest_100_percent_gpu_sample_run"] == 0
assert all(load["checks"].values())
assert manifest["authority"] == {
"actuation_allowed": False,
"candidate_accepted": False,
"commands_enabled": False,
"ground_truth": False,
"navigation_or_safety_accepted": False,
}
tests/test_m48s_yolox_all_coco_result.py
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import json
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
RESULT_ID = (
"m48s-yolox-all-coco-shadow-"
"7dbe6043b3fc12c7ddb162f609f883d86b34a4f2dd3785a632795f257e192d06"
)
RESULT_ROOT = (
REPOSITORY_ROOT
/ ".runtime/compute-experiments/m48s-semantic-shadow/yolox-all-coco-results"
/ RESULT_ID
)
def test_all_coco_yolox_uses_one_inference_pass_with_bounded_postprocess_cost() -> None:
manifest = json.loads((RESULT_ROOT / "manifest.json").read_text("utf-8"))
metrics = manifest["metrics"]
assert manifest["result_id"] == RESULT_ID
assert manifest["completed"] is True
assert manifest["accepted"] is False
assert metrics["frames"] == {"completed": 11, "requested": 11}
assert metrics["inference_passes_per_frame"] == 1
assert metrics["frozen_detection_count"] == 44
assert metrics["all_coco_detection_count"] == 45
assert metrics["added_detection_count"] == 1
assert metrics["all_coco_class_counts"] == {
"car": 36,
"handbag": 1,
"person": 3,
"truck": 5,
}
benchmark = metrics["postprocess_benchmark"]
assert benchmark["iterations_per_profile_per_frame"] == 20
frozen_mean = benchmark["timing_ms"]["frozen_ms"]["mean"]
all_coco_mean = benchmark["timing_ms"]["all_coco_ms"]["mean"]
assert all_coco_mean - frozen_mean < 1.0
assert metrics["all_coco_core_capacity_fps"] > 30.0
assert manifest["authority"]["commands_enabled"] is False
assert manifest["authority"]["navigation_or_safety_accepted"] is False
tests/test_rf_detr_detector_provider.py
+202
View File
@@ -0,0 +1,202 @@
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import cast
import numpy as np
import pytest
from numpy.typing import NDArray
from k1link.perception.contracts import (
ClockBasis,
ModalityOutcome,
ModalityStatus,
SourceEnvelope,
TimestampBundle,
)
from k1link.perception.detector import (
RF_DETR_SHADOW_MODEL_ID,
RF_DETR_SHADOW_PREPROCESS_ID,
RF_DETR_SHADOW_PROVIDER_ID,
RfDetrShadowDetectorProvider,
)
from k1link.perception.providers import SourcePacket
from k1link.perception.rf_detr_object_detector import (
RF_DETR_CONFIG,
RF_DETR_ENGINE_SHA256,
RF_DETR_FP16_ONNX_SHA256,
RfDetrConfig,
RfDetrDetectorError,
RfDetrRawOutput,
TritonRfDetrHttpInferenceBackend,
postprocess_rf_detr,
preprocess_raw_kb4_rf_detr,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
def _status() -> ModalityStatus:
return ModalityStatus(True, ModalityOutcome.AVAILABLE, "test-available")
def _packet(sequence: int, image: object) -> SourcePacket:
return SourcePacket(
envelope=SourceEnvelope(
source_id="RAVNOVES00",
session_id="20260720T065719Z_viewer_live",
frame_id=f"frame-{sequence:06d}",
sequence=sequence,
timestamps=TimestampBundle(
utc_ns=1_000 + sequence,
monotonic_ns=2_000 + sequence,
source_ns=3_000 + sequence,
clock_basis=ClockBasis.RECORDED_HOST,
),
source_age_ns=0,
binding_reason="test-recorded-source",
calibration_id="camera-1-kb4-test",
representation_id="registered-map-increment-v1",
image=_status(),
registered_point_increment=_status(),
pose=_status(),
),
image_payload=image,
registered_point_increment_payload=("points", sequence),
pose_payload=("pose", sequence),
)
class _Resizer:
def __init__(self) -> None:
self.source: NDArray[np.uint8] | None = None
def resize(
self, image: NDArray[np.uint8], width: int, height: int
) -> NDArray[np.uint8]:
self.source = image.copy()
output = np.empty((height, width, 3), dtype=np.uint8)
output[:, :, 0] = 255
output[:, :, 1] = 0
output[:, :, 2] = 127
return output
class _Backend:
def __init__(self, output: RfDetrRawOutput) -> None:
self.output = output
self.calls = 0
def infer(self, tensor: NDArray[np.float32]) -> RfDetrRawOutput:
assert tensor.shape == (1, 3, 704, 704)
assert tensor.dtype == np.float32
self.calls += 1
return self.output
def _output() -> RfDetrRawOutput:
boxes = np.zeros((1, 300, 4), dtype=np.float16)
logits = np.full((1, 300, 91), -20.0, dtype=np.float16)
boxes[0, 0] = (0.5, 0.5, 0.25, np.float16(1 / 3))
logits[0, 0, 18] = np.float16(math.log(3.0)) # dog, score 0.75
boxes[0, 1] = (0.25, 0.25, 0.1, 0.2)
logits[0, 1, 1] = np.float16(math.log(4.0)) # person, score 0.80
boxes[0, 2] = (0.75, 0.25, 0.1, 0.2)
logits[0, 2, 62] = np.float16(math.log(9.0)) # chair, non-risk
logits[0, 3, 12] = np.float16(math.log(9.0)) # unused COCO slot
return RfDetrRawOutput(boxes=boxes, logits=logits)
def test_preprocess_masks_bgr_converts_rgb_stretches_and_normalizes() -> None:
image = np.zeros((600, 800, 3), dtype=np.uint8)
image[:, :] = (10, 20, 30)
mask = np.ones((600, 800), dtype=np.bool_)
mask[0, 0] = False
resizer = _Resizer()
tensor = preprocess_raw_kb4_rf_detr(image, mask, resizer=resizer)
assert resizer.source is not None
assert tuple(resizer.source[1, 1]) == (30, 20, 10)
assert tuple(resizer.source[0, 0]) == (114, 114, 114)
assert tensor.shape == (1, 3, 704, 704)
assert tensor.dtype == np.float32
assert tensor[0, 0, 0, 0] == pytest.approx((1.0 - 0.485) / 0.229)
assert tensor[0, 1, 0, 0] == pytest.approx((0.0 - 0.456) / 0.224)
assert tensor[0, 2, 0, 0] == pytest.approx((127 / 255.0 - 0.406) / 0.225)
def test_postprocess_maps_sparse_coco_slots_and_emits_only_risk_classes() -> None:
result = postprocess_rf_detr(_output(), np.ones((600, 800), dtype=np.bool_))
assert tuple(item.label for item in result.detections) == ("person", "dog")
assert result.detections[0].score == pytest.approx(0.8, abs=0.001)
assert result.detections[1].score == pytest.approx(0.75, abs=0.001)
assert result.detections[1].bbox_xyxy == pytest.approx(
(300.0, 200.0, 500.0, 400.0), abs=0.03
)
assert dict(result.rejected) == {"non-risk-class": 1, "unmapped-class-slot": 1}
with pytest.raises(RfDetrDetectorError, match="tensor types"):
postprocess_rf_detr(
RfDetrRawOutput(
boxes=cast(NDArray[np.float16], _output().boxes.astype(np.float32)),
logits=_output().logits,
),
np.ones((600, 800), dtype=np.bool_),
)
def test_shadow_provider_uses_one_pass_and_preserves_semantic_hints() -> None:
backend = _Backend(_output())
provider = RfDetrShadowDetectorProvider(
mask=np.ones((600, 800), dtype=np.bool_),
backend=backend,
resizer=_Resizer(),
clock_ns=iter((10, 30)).__next__,
)
proposals = provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8)))
assert backend.calls == 1
assert tuple(item.semantic_hint for item in proposals) == ("person", "dog")
assert all(item.provider_id == RF_DETR_SHADOW_PROVIDER_ID for item in proposals)
assert all(item.model_id == RF_DETR_SHADOW_MODEL_ID for item in proposals)
assert all(item.preprocess_id == RF_DETR_SHADOW_PREPROCESS_ID for item in proposals)
assert provider.snapshot().completed_frames == 1
assert provider.snapshot().proposal_count == 2
assert provider.snapshot().core_duration_ns == 20
def test_shadow_profile_is_fixed_and_transport_pins_model_version() -> None:
assert RF_DETR_CONFIG.minimum_score == 0.25
with pytest.raises(RfDetrDetectorError, match="cannot be tuned"):
RfDetrConfig(minimum_score=0.5)
backend = TritonRfDetrHttpInferenceBackend("http://127.0.0.1:8100")
try:
assert backend.path == "/v2/models/rf_detr_large/versions/1/infer"
finally:
backend.close()
with pytest.raises(RfDetrDetectorError, match="explicit HTTP origin"):
TritonRfDetrHttpInferenceBackend("http://user:secret@127.0.0.1:8100")
def test_shadow_profile_pins_worker_engine_and_retains_false_authority() -> None:
profile = json.loads(
(REPOSITORY_ROOT / "config/perception/rf-detr-large-risk-shadow-v0.json").read_text(
"utf-8"
)
)
assert profile["model"]["strongly_typed_fp16_onnx_sha256"] == RF_DETR_FP16_ONNX_SHA256
assert (
profile["model"]["worker_006_rtx4090_tensorrt_11_engine_sha256"]
== RF_DETR_ENGINE_SHA256
)
assert profile["emission"]["single_inference_per_source_frame"] is True
assert profile["emission"]["geometry_owns_static_occupancy"] is True
assert profile["emission"]["unlisted_semantic_classes_emitted"] is False
assert not any(profile["authority"].values())
tests/test_yolox_all_coco_profile.py
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
import json
from pathlib import Path
from k1link.perception.detector import ALL_COCO_YOLOX_PROVIDER_ID
from k1link.perception.yolox_object_detector import (
ALL_COCO_CLASS_IDS,
ALL_COCO_YOLOX_CONFIG,
COCO_CLASSES,
YOLOX_MODEL_SHA256,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
PROFILE_PATH = REPOSITORY_ROOT / "config/perception/yolox-s-all-coco-shadow-v2.json"
def test_all_coco_profile_matches_executable_provider_contract() -> None:
profile = json.loads(PROFILE_PATH.read_text("utf-8"))
assert profile["schema_version"] == "missioncore.yolox-detector-profile/v2"
assert profile["provider_id"] == ALL_COCO_YOLOX_PROVIDER_ID
assert profile["model"]["model_sha256"] == YOLOX_MODEL_SHA256
assert profile["model"]["additional_inference_passes"] == 0
assert tuple(profile["postprocess"]["target_class_ids"]) == ALL_COCO_CLASS_IDS
assert ALL_COCO_YOLOX_CONFIG.target_class_ids == tuple(range(len(COCO_CLASSES)))
assert "dog" in profile["class_policy"]["risk_groups"]["animal"]
assert profile["authority"]["commands_enabled"] is False
assert profile["authority"]["navigation_or_safety_accepted"] is False
tests/test_yolox_detector_provider.py
+47
View File
@@ -17,13 +17,19 @@ from k1link.perception.contracts import (
TimestampBundle,
)
from k1link.perception.detector import (
ALL_COCO_YOLOX_PROVIDER_ID,
FROZEN_YOLOX_PROVIDER_ID,
AllCocoYoloxDetectorProvider,
DetectorProviderError,
FrozenYoloxDetectorProvider,
proposals_from_detections,
)
from k1link.perception.providers import SourcePacket
from k1link.perception.yolox_object_detector import (
ALL_COCO_CLASS_IDS,
ALL_COCO_YOLOX_CONFIG,
COCO_CLASSES,
AllCocoYoloxConfig,
FrozenYoloxConfig,
TritonHttpInferenceBackend,
YoloxDetection,
@@ -102,6 +108,14 @@ def _one_person_output() -> NDArray[np.float32]:
return output
def _one_dog_output() -> NDArray[np.float32]:
output = np.zeros((1, 8400, 85), dtype=np.float32)
output[0, 0, :4] = [40.0, 30.0, math.log(10.0), math.log(10.0)]
output[0, 0, 4] = 0.9
output[0, 0, 5 + 16] = 0.9
return output
def test_frozen_preprocess_and_postprocess_match_the_e46j_contract() -> None:
image = np.full((600, 800, 3), 7, dtype=np.uint8)
mask = np.ones((600, 800), dtype=np.bool_)
@@ -174,6 +188,39 @@ def test_frozen_profile_rejects_in_place_threshold_tuning() -> None:
FrozenYoloxConfig(minimum_score=0.51)
def test_all_coco_profile_emits_dog_without_another_inference_pass() -> None:
mask = np.ones((600, 800), dtype=np.bool_)
assert postprocess_yolox(_one_dog_output(), mask).detections == ()
all_coco = postprocess_yolox(
_one_dog_output(),
mask,
config=ALL_COCO_YOLOX_CONFIG,
)
backend = _Backend(_one_dog_output())
provider = AllCocoYoloxDetectorProvider(
mask=mask,
backend=backend,
resizer=_Resizer(),
)
proposals = provider.detect(
_packet(16, np.zeros((600, 800, 3), dtype=np.uint8))
)
assert tuple(range(80)) == ALL_COCO_CLASS_IDS
assert len(COCO_CLASSES) == 80
assert tuple(item.label for item in all_coco.detections) == ("dog",)
assert backend.calls == 1
assert len(proposals) == 1
assert proposals[0].provider_id == ALL_COCO_YOLOX_PROVIDER_ID
assert proposals[0].semantic_hint == "dog"
def test_all_coco_profile_is_versioned_and_cannot_be_tuned_in_place() -> None:
with pytest.raises(YoloxDetectorError, match="all-COCO.*cannot be tuned"):
AllCocoYoloxConfig(target_class_ids=(0, 16))
def test_triton_transport_pins_the_frozen_model_version() -> None:
backend = TritonHttpInferenceBackend("http://127.0.0.1:8000")
try: