feat(perception): integrate native RF-DETR shadow provider
This commit is contained in:
DCCONSTRUCTIONS
@@ -14,6 +14,15 @@ from numpy.typing import NDArray
|
||||
|
||||
from .contracts import BoundingRegion2D, ObjectProposal2D
|
||||
from .providers import SourcePacket
|
||||
from .rf_detr_native_object_detector import (
|
||||
RF_DETR_NATIVE_CONFIG,
|
||||
RF_DETR_NATIVE_MODEL_ID,
|
||||
RF_DETR_NATIVE_MODEL_VERSION,
|
||||
NativeRfDetrConfig,
|
||||
NativeRfDetrInferenceBackend,
|
||||
postprocess_native_rf_detr,
|
||||
prepare_raw_kb4_rf_detr_native,
|
||||
)
|
||||
from .rf_detr_object_detector import (
|
||||
RF_DETR_CONFIG,
|
||||
RF_DETR_MODEL_ID,
|
||||
@@ -45,6 +54,15 @@ FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
|
||||
RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
|
||||
RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}"
|
||||
RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0"
|
||||
RF_DETR_NATIVE_SHADOW_PROVIDER_ID: Final = (
|
||||
"triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0"
|
||||
)
|
||||
RF_DETR_NATIVE_SHADOW_MODEL_ID: Final = (
|
||||
f"{RF_DETR_NATIVE_MODEL_ID}:{RF_DETR_NATIVE_MODEL_VERSION}"
|
||||
)
|
||||
RF_DETR_NATIVE_SHADOW_PREPROCESS_ID: Final = (
|
||||
"raw-kb4-uint8-fused-mask-rgb-pad8-imagenet-trt/v0"
|
||||
)
|
||||
|
||||
|
||||
class DetectorProviderError(RuntimeError):
|
||||
@@ -414,6 +432,164 @@ def proposals_from_rf_detr_detections(
|
||||
)
|
||||
|
||||
|
||||
class NativeRfDetrShadowDetectorProvider:
|
||||
"""Emit risk classes from one exact-raster native RF-DETR inference pass."""
|
||||
|
||||
provider_id: str = RF_DETR_NATIVE_SHADOW_PROVIDER_ID
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
mask: NDArray[np.bool_],
|
||||
backend: NativeRfDetrInferenceBackend,
|
||||
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
|
||||
clock_ns: Callable[[], int] = time.perf_counter_ns,
|
||||
timing_observer: DetectorTimingObserver | None = None,
|
||||
) -> None:
|
||||
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
|
||||
raise DetectorProviderError("native RF-DETR valid-FOV mask is incompatible")
|
||||
self.mask = np.asarray(mask, dtype=np.bool_)
|
||||
self.backend = backend
|
||||
self.config = config
|
||||
self._clock_ns = clock_ns
|
||||
self.timing_observer = timing_observer
|
||||
self._lock = Lock()
|
||||
self._input_frames = 0
|
||||
self._completed_frames = 0
|
||||
self._failed_frames = 0
|
||||
self._zero_proposal_frames = 0
|
||||
self._proposal_count = 0
|
||||
self._rejected: Counter[str] = Counter()
|
||||
self._core_duration_ns = 0
|
||||
self._warmup_started = False
|
||||
self._warmup_snapshot: DetectorWarmupSnapshot | None = None
|
||||
|
||||
def warm_up(self) -> DetectorWarmupSnapshot:
|
||||
"""Prime raw transport and postprocessing before source admission."""
|
||||
|
||||
with self._lock:
|
||||
if self._warmup_snapshot is not None:
|
||||
return self._warmup_snapshot
|
||||
if self._warmup_started:
|
||||
raise DetectorProviderError("native RF-DETR warmup is already in progress")
|
||||
self._warmup_started = True
|
||||
started_ns = int(self._clock_ns())
|
||||
try:
|
||||
image = np.zeros(
|
||||
(self.config.source_height, self.config.source_width, 3),
|
||||
dtype=np.uint8,
|
||||
)
|
||||
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
|
||||
preprocessed_ns = int(self._clock_ns())
|
||||
output = self.backend.infer(tensor)
|
||||
inferred_ns = int(self._clock_ns())
|
||||
postprocess_native_rf_detr(output, self.mask, config=self.config)
|
||||
completed_ns = int(self._clock_ns())
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._warmup_started = False
|
||||
raise
|
||||
snapshot = DetectorWarmupSnapshot(
|
||||
completed=True,
|
||||
inference_passes=1,
|
||||
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
|
||||
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
|
||||
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
|
||||
total_duration_ns=max(0, completed_ns - started_ns),
|
||||
)
|
||||
with self._lock:
|
||||
self._warmup_snapshot = snapshot
|
||||
return snapshot
|
||||
|
||||
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
|
||||
payload = packet.image_payload
|
||||
with self._lock:
|
||||
self._input_frames += 1
|
||||
started_ns = int(self._clock_ns())
|
||||
try:
|
||||
if not isinstance(payload, np.ndarray):
|
||||
raise DetectorProviderError(
|
||||
"native RF-DETR requires a decoded BGR image payload"
|
||||
)
|
||||
image = np.asarray(payload)
|
||||
if image.dtype != np.uint8:
|
||||
raise DetectorProviderError("decoded BGR image must be uint8")
|
||||
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
|
||||
preprocessed_ns = (
|
||||
int(self._clock_ns()) if self.timing_observer is not None else started_ns
|
||||
)
|
||||
output = self.backend.infer(tensor)
|
||||
inferred_ns = (
|
||||
int(self._clock_ns()) if self.timing_observer is not None else preprocessed_ns
|
||||
)
|
||||
postprocessed = postprocess_native_rf_detr(
|
||||
output,
|
||||
self.mask,
|
||||
config=self.config,
|
||||
)
|
||||
proposals = proposals_from_native_rf_detr_detections(
|
||||
packet,
|
||||
postprocessed.detections,
|
||||
)
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._failed_frames += 1
|
||||
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
|
||||
raise
|
||||
completed_ns = int(self._clock_ns())
|
||||
with self._lock:
|
||||
self._completed_frames += 1
|
||||
self._proposal_count += len(proposals)
|
||||
self._zero_proposal_frames += not proposals
|
||||
self._rejected.update(dict(postprocessed.rejected))
|
||||
self._core_duration_ns += max(0, completed_ns - started_ns)
|
||||
if self.timing_observer is not None:
|
||||
self.timing_observer(
|
||||
DetectorFrameTiming(
|
||||
sequence=packet.envelope.sequence,
|
||||
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
|
||||
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
|
||||
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
|
||||
total_duration_ns=max(0, completed_ns - started_ns),
|
||||
)
|
||||
)
|
||||
return proposals
|
||||
|
||||
def snapshot(self) -> DetectorProviderSnapshot:
|
||||
with self._lock:
|
||||
return DetectorProviderSnapshot(
|
||||
input_frames=self._input_frames,
|
||||
completed_frames=self._completed_frames,
|
||||
failed_frames=self._failed_frames,
|
||||
zero_proposal_frames=self._zero_proposal_frames,
|
||||
proposal_count=self._proposal_count,
|
||||
rejected=tuple(sorted(self._rejected.items())),
|
||||
core_duration_ns=self._core_duration_ns,
|
||||
)
|
||||
|
||||
|
||||
def proposals_from_native_rf_detr_detections(
|
||||
packet: SourcePacket,
|
||||
detections: tuple[RfDetrDetection, ...],
|
||||
) -> tuple[ObjectProposal2D, ...]:
|
||||
envelope = packet.envelope
|
||||
return tuple(
|
||||
ObjectProposal2D(
|
||||
proposal_id=f"proposal-{envelope.sequence}-{index}",
|
||||
source_id=envelope.source_id,
|
||||
frame_id=envelope.frame_id,
|
||||
region=BoundingRegion2D(*detection.bbox_xyxy),
|
||||
objectness=detection.score,
|
||||
provider_id=RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
|
||||
model_id=RF_DETR_NATIVE_SHADOW_MODEL_ID,
|
||||
preprocess_id=RF_DETR_NATIVE_SHADOW_PREPROCESS_ID,
|
||||
semantic_hint=detection.label,
|
||||
provider_tracklet=None,
|
||||
)
|
||||
for index, detection in enumerate(detections)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ALL_COCO_YOLOX_PROVIDER_ID",
|
||||
"FROZEN_YOLOX_MODEL_ID",
|
||||
@@ -422,6 +598,9 @@ __all__ = [
|
||||
"RF_DETR_SHADOW_MODEL_ID",
|
||||
"RF_DETR_SHADOW_PREPROCESS_ID",
|
||||
"RF_DETR_SHADOW_PROVIDER_ID",
|
||||
"RF_DETR_NATIVE_SHADOW_MODEL_ID",
|
||||
"RF_DETR_NATIVE_SHADOW_PREPROCESS_ID",
|
||||
"RF_DETR_NATIVE_SHADOW_PROVIDER_ID",
|
||||
"DetectorProviderError",
|
||||
"DetectorProviderSnapshot",
|
||||
"DetectorFrameTiming",
|
||||
@@ -429,7 +608,9 @@ __all__ = [
|
||||
"DetectorWarmupSnapshot",
|
||||
"AllCocoYoloxDetectorProvider",
|
||||
"FrozenYoloxDetectorProvider",
|
||||
"NativeRfDetrShadowDetectorProvider",
|
||||
"RfDetrShadowDetectorProvider",
|
||||
"proposals_from_detections",
|
||||
"proposals_from_native_rf_detr_detections",
|
||||
"proposals_from_rf_detr_detections",
|
||||
]
|
||||
|
||||
@@ -8,12 +8,15 @@ from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from threading import Event
|
||||
from typing import Literal
|
||||
|
||||
from .baseline import load_m4_baseline
|
||||
from .detector import (
|
||||
RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
|
||||
RF_DETR_SHADOW_PROVIDER_ID,
|
||||
DetectorTimingObserver,
|
||||
DetectorWarmupSnapshot,
|
||||
NativeRfDetrShadowDetectorProvider,
|
||||
RfDetrShadowDetectorProvider,
|
||||
)
|
||||
from .geometry import (
|
||||
@@ -41,6 +44,12 @@ from .recorded_source import (
|
||||
SourcePacingObserver,
|
||||
)
|
||||
from .reference_graph_runtime import ReferenceGraphRuntimePaths
|
||||
from .rf_detr_native_object_detector import (
|
||||
RF_DETR_NATIVE_ENGINE_SHA256,
|
||||
RF_DETR_NATIVE_MODEL_ID,
|
||||
RF_DETR_NATIVE_MODEL_VERSION,
|
||||
TritonNativeRfDetrHttpInferenceBackend,
|
||||
)
|
||||
from .rf_detr_object_detector import (
|
||||
RF_DETR_ENGINE_SHA256,
|
||||
RF_DETR_MODEL_ID,
|
||||
@@ -66,13 +75,18 @@ class M48sReferenceGraphRuntime:
|
||||
"""Own one RF-DETR shadow graph and its persistent inference transport."""
|
||||
|
||||
graph: ReferencePerceptionGraphV2
|
||||
inference_backend: TritonRfDetrHttpInferenceBackend
|
||||
inference_backend: (
|
||||
TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend
|
||||
)
|
||||
source_prefetch: PrefetchedRecordedImageDecoder
|
||||
_preparation_stop_event: Event = field(default_factory=Event)
|
||||
|
||||
def warm_up_detector(self) -> DetectorWarmupSnapshot:
|
||||
detector = self.graph.detector
|
||||
if not isinstance(detector, RfDetrShadowDetectorProvider):
|
||||
if not isinstance(
|
||||
detector,
|
||||
(RfDetrShadowDetectorProvider, NativeRfDetrShadowDetectorProvider),
|
||||
):
|
||||
raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup")
|
||||
return detector.warm_up()
|
||||
|
||||
@@ -125,7 +139,7 @@ def build_m48s_reference_graph_runtime(
|
||||
ProviderRole.THREAT: paths.threat_profile,
|
||||
}
|
||||
_validate_provider_digests(config, pinned_files)
|
||||
_validate_detector_profile(detector_profile)
|
||||
detector_variant = _validate_detector_profile(detector_profile)
|
||||
|
||||
load_m4_baseline(paths.baseline_profile)
|
||||
geometry_profile = load_geometry_profile(paths.geometry_profile)
|
||||
@@ -161,7 +175,22 @@ def build_m48s_reference_graph_runtime(
|
||||
)
|
||||
if maximum_frames is not None:
|
||||
source = _LimitedSource(source, maximum_frames)
|
||||
backend = TritonRfDetrHttpInferenceBackend(triton_origin)
|
||||
if detector_variant == "legacy-704":
|
||||
backend: (
|
||||
TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend
|
||||
) = TritonRfDetrHttpInferenceBackend(triton_origin)
|
||||
detector = RfDetrShadowDetectorProvider(
|
||||
mask=load_valid_fov_mask(paths.valid_fov_mask),
|
||||
backend=backend,
|
||||
timing_observer=detector_timing_observer,
|
||||
)
|
||||
else:
|
||||
backend = TritonNativeRfDetrHttpInferenceBackend(triton_origin)
|
||||
detector = NativeRfDetrShadowDetectorProvider(
|
||||
mask=load_valid_fov_mask(paths.valid_fov_mask),
|
||||
backend=backend,
|
||||
timing_observer=detector_timing_observer,
|
||||
)
|
||||
try:
|
||||
store = RecordedGeometryStore(
|
||||
source_pack_path=paths.source_pack,
|
||||
@@ -175,11 +204,7 @@ def build_m48s_reference_graph_runtime(
|
||||
graph = ReferencePerceptionGraphV2(
|
||||
config=config,
|
||||
source=source,
|
||||
detector=RfDetrShadowDetectorProvider(
|
||||
mask=load_valid_fov_mask(paths.valid_fov_mask),
|
||||
backend=backend,
|
||||
timing_observer=detector_timing_observer,
|
||||
),
|
||||
detector=detector,
|
||||
geometry=Ravnoves00GeometryAssociationProvider(store=store),
|
||||
temporal=BoundedSpatialTemporalProvider(
|
||||
point_resolver=store,
|
||||
@@ -249,7 +274,7 @@ def _validate_provider_digests(
|
||||
raise M48sReferenceGraphRuntimeError(f"{role.value} provider profile digest changed")
|
||||
|
||||
|
||||
def _validate_detector_profile(path: Path) -> None:
|
||||
def _validate_detector_profile(path: Path) -> Literal["legacy-704", "native-kb4"]:
|
||||
try:
|
||||
document = json.loads(path.resolve(strict=True).read_text("utf-8"))
|
||||
model = document["model"]
|
||||
@@ -257,24 +282,46 @@ def _validate_detector_profile(path: Path) -> None:
|
||||
authority = document["authority"]
|
||||
except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc:
|
||||
raise M48sReferenceGraphRuntimeError("RF-DETR profile is incomplete") from exc
|
||||
if (
|
||||
document.get("schema_version") != "missioncore.rf-detr-risk-shadow-profile/v0"
|
||||
or document.get("provider_id") != RF_DETR_SHADOW_PROVIDER_ID
|
||||
or model.get("model_id") != RF_DETR_MODEL_ID
|
||||
or model.get("model_version") != RF_DETR_MODEL_VERSION
|
||||
or model.get("worker_006_rtx4090_tensorrt_11_engine_sha256") != RF_DETR_ENGINE_SHA256
|
||||
or status.get("detector_load_gate_passed") is not True
|
||||
or status.get("production_accepted") is not False
|
||||
or any(
|
||||
authority.get(key) is not False
|
||||
for key in (
|
||||
"candidate_accepted",
|
||||
"commands_enabled",
|
||||
"actuation_allowed",
|
||||
"navigation_or_safety_accepted",
|
||||
)
|
||||
authority_false = not any(
|
||||
authority.get(key) is not False
|
||||
for key in (
|
||||
"candidate_accepted",
|
||||
"commands_enabled",
|
||||
"actuation_allowed",
|
||||
"navigation_or_safety_accepted",
|
||||
)
|
||||
):
|
||||
)
|
||||
legacy = (
|
||||
document.get("schema_version") == "missioncore.rf-detr-risk-shadow-profile/v0"
|
||||
and document.get("provider_id") == RF_DETR_SHADOW_PROVIDER_ID
|
||||
and model.get("model_id") == RF_DETR_MODEL_ID
|
||||
and model.get("model_version") == RF_DETR_MODEL_VERSION
|
||||
and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256")
|
||||
== RF_DETR_ENGINE_SHA256
|
||||
and status.get("detector_load_gate_passed") is True
|
||||
and status.get("production_accepted") is False
|
||||
and authority_false
|
||||
)
|
||||
native = (
|
||||
document.get("schema_version")
|
||||
== "missioncore.rf-detr-native-risk-shadow-profile/v0"
|
||||
and document.get("provider_id") == RF_DETR_NATIVE_SHADOW_PROVIDER_ID
|
||||
and model.get("model_id") == RF_DETR_NATIVE_MODEL_ID
|
||||
and model.get("model_version") == RF_DETR_NATIVE_MODEL_VERSION
|
||||
and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256")
|
||||
== RF_DETR_NATIVE_ENGINE_SHA256
|
||||
and status.get("native_tensor_parity_passed") is True
|
||||
and status.get("full_ravnoves00_runtime_gate_passed") is True
|
||||
and status.get("legacy_704_box_agreement_gate_passed") is False
|
||||
and status.get("integrated_world_state_gate_passed") is False
|
||||
and status.get("production_accepted") is False
|
||||
and authority_false
|
||||
)
|
||||
if legacy:
|
||||
return "legacy-704"
|
||||
if native:
|
||||
return "native-kb4"
|
||||
else:
|
||||
raise M48sReferenceGraphRuntimeError("RF-DETR shadow profile identity changed")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,364 @@
|
||||
"""Native raw-KB4 RF-DETR-L TensorRT transport and postprocessing.
|
||||
|
||||
The TensorRT engine owns valid-FOV masking, BGR-to-RGB conversion, eight
|
||||
bottom padding rows and ImageNet normalization. The client sends the exact
|
||||
800x600 UINT8 KB4 raster and performs no geometric resampling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import urllib.parse
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Protocol, cast
|
||||
|
||||
import numpy as np
|
||||
from numpy.typing import NDArray
|
||||
|
||||
from .rf_detr_object_detector import (
|
||||
COCO_SPARSE_TO_CONTIGUOUS,
|
||||
RISK_CLASS_IDS,
|
||||
RfDetrDetection,
|
||||
RfDetrPostprocessResult,
|
||||
RfDetrRawOutput,
|
||||
)
|
||||
from .yolox_object_detector import COCO_CLASSES, YOLOX_VALID_FOV_SHA256
|
||||
|
||||
RF_DETR_NATIVE_MODEL_ID: Final = "rf_detr_large_native_kb4"
|
||||
RF_DETR_NATIVE_MODEL_VERSION: Final = 1
|
||||
RF_DETR_NATIVE_CHECKPOINT_SHA256: Final = (
|
||||
"0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38"
|
||||
)
|
||||
RF_DETR_NATIVE_CORE_ONNX_SHA256: Final = (
|
||||
"62e549748a1d17646b90ad06d3ac8a1b79595b7e9270cac4564418f023079176"
|
||||
)
|
||||
RF_DETR_NATIVE_FP16_ONNX_SHA256: Final = (
|
||||
"00b29fa2ff3d5fca730ebf3b8c33b10e9d690cc97bbbbfa5563d6e0abf210999"
|
||||
)
|
||||
RF_DETR_NATIVE_WRAPPED_ONNX_SHA256: Final = (
|
||||
"acdd01623a00d100331473c0a99eab1e5adf33117cbab900c8ae078edd4aa346"
|
||||
)
|
||||
RF_DETR_NATIVE_ENGINE_SHA256: Final = (
|
||||
"b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695"
|
||||
)
|
||||
RF_DETR_NATIVE_VALID_FOV_SHA256: Final = YOLOX_VALID_FOV_SHA256
|
||||
|
||||
|
||||
class NativeRfDetrDetectorError(RuntimeError):
|
||||
"""The native RF-DETR profile, tensor or response is incompatible."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeRfDetrConfig:
|
||||
source_width: int = 800
|
||||
source_height: int = 600
|
||||
model_width: int = 800
|
||||
model_height: int = 608
|
||||
bottom_padding_rows: int = 8
|
||||
fill_value: int = 114
|
||||
minimum_score: float = 0.25
|
||||
target_class_ids: tuple[int, ...] = RISK_CLASS_IDS
|
||||
maximum_detections: int = 300
|
||||
minimum_box_area_pixels: float = 64.0
|
||||
maximum_box_area_fraction: float = 0.5
|
||||
minimum_valid_fov_fraction: float = 0.5
|
||||
require_center_inside_valid_fov: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
self.source_width,
|
||||
self.source_height,
|
||||
self.model_width,
|
||||
self.model_height,
|
||||
self.bottom_padding_rows,
|
||||
self.fill_value,
|
||||
self.minimum_score,
|
||||
self.target_class_ids,
|
||||
self.maximum_detections,
|
||||
self.minimum_box_area_pixels,
|
||||
self.maximum_box_area_fraction,
|
||||
self.minimum_valid_fov_fraction,
|
||||
self.require_center_inside_valid_fov,
|
||||
) != (
|
||||
800,
|
||||
600,
|
||||
800,
|
||||
608,
|
||||
8,
|
||||
114,
|
||||
0.25,
|
||||
RISK_CLASS_IDS,
|
||||
300,
|
||||
64.0,
|
||||
0.5,
|
||||
0.5,
|
||||
True,
|
||||
):
|
||||
raise NativeRfDetrDetectorError(
|
||||
"native RF-DETR shadow profile cannot be tuned in place"
|
||||
)
|
||||
|
||||
|
||||
RF_DETR_NATIVE_CONFIG: Final = NativeRfDetrConfig()
|
||||
|
||||
|
||||
class NativeRfDetrInferenceBackend(Protocol):
|
||||
def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput: ...
|
||||
|
||||
|
||||
class TritonNativeRfDetrHttpInferenceBackend:
|
||||
"""Persistent Triton V2 HTTP transport for exact raw UINT8 KB4 frames."""
|
||||
|
||||
def __init__(self, endpoint: str, *, timeout_seconds: float = 60.0) -> None:
|
||||
parsed = urllib.parse.urlsplit(endpoint)
|
||||
if (
|
||||
parsed.scheme != "http"
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton endpoint must be an explicit HTTP origin"
|
||||
)
|
||||
if not math.isfinite(timeout_seconds) or timeout_seconds <= 0:
|
||||
raise NativeRfDetrDetectorError("Triton timeout must be positive")
|
||||
self.path = (
|
||||
f"{parsed.path.rstrip('/')}/v2/models/{RF_DETR_NATIVE_MODEL_ID}"
|
||||
f"/versions/{RF_DETR_NATIVE_MODEL_VERSION}/infer"
|
||||
)
|
||||
self.connection = http.client.HTTPConnection(
|
||||
parsed.hostname,
|
||||
parsed.port or 80,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self.connection.close()
|
||||
|
||||
def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput:
|
||||
contiguous = np.ascontiguousarray(tensor, dtype=np.uint8)
|
||||
if contiguous.shape != (1, 600, 800, 3):
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR input tensor is incompatible"
|
||||
)
|
||||
binary = contiguous.tobytes()
|
||||
header = {
|
||||
"inputs": [
|
||||
{
|
||||
"name": "raw_kb4_bgr",
|
||||
"shape": [1, 600, 800, 3],
|
||||
"datatype": "UINT8",
|
||||
"parameters": {"binary_data_size": len(binary)},
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{"name": "dets", "parameters": {"binary_data": True}},
|
||||
{"name": "labels", "parameters": {"binary_data": True}},
|
||||
],
|
||||
}
|
||||
encoded = json.dumps(header, sort_keys=True, separators=(",", ":")).encode()
|
||||
self.connection.request(
|
||||
"POST",
|
||||
self.path,
|
||||
body=encoded + binary,
|
||||
headers={
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Inference-Header-Content-Length": str(len(encoded)),
|
||||
},
|
||||
)
|
||||
response = self.connection.getresponse()
|
||||
payload = response.read()
|
||||
if response.status != 200:
|
||||
raise NativeRfDetrDetectorError(
|
||||
f"Triton native RF-DETR inference failed with HTTP {response.status}"
|
||||
)
|
||||
header_value = response.getheader("Inference-Header-Content-Length")
|
||||
try:
|
||||
header_length = int(header_value or "")
|
||||
descriptor = json.loads(payload[:header_length])
|
||||
outputs = descriptor["outputs"]
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR output descriptor is invalid"
|
||||
) from exc
|
||||
if not isinstance(outputs, list) or len(outputs) != 2:
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR output count changed"
|
||||
)
|
||||
offset = header_length
|
||||
arrays: dict[str, NDArray[np.float16]] = {}
|
||||
for output, expected_name, expected_shape in zip(
|
||||
outputs,
|
||||
("dets", "labels"),
|
||||
((1, 300, 4), (1, 300, 91)),
|
||||
strict=True,
|
||||
):
|
||||
try:
|
||||
name = output["name"]
|
||||
datatype = output["datatype"]
|
||||
shape = tuple(int(value) for value in output["shape"])
|
||||
byte_length = int(output["parameters"]["binary_data_size"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR output descriptor is incomplete"
|
||||
) from exc
|
||||
expected_bytes = math.prod(expected_shape) * np.dtype("<f2").itemsize
|
||||
if (
|
||||
name != expected_name
|
||||
or datatype != "FP16"
|
||||
or shape != expected_shape
|
||||
or byte_length != expected_bytes
|
||||
or offset + byte_length > len(payload)
|
||||
):
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR output identity changed"
|
||||
)
|
||||
array = np.frombuffer(payload[offset : offset + byte_length], dtype="<f2")
|
||||
arrays[name] = np.asarray(array.reshape(shape), dtype=np.float16)
|
||||
offset += byte_length
|
||||
if offset != len(payload):
|
||||
raise NativeRfDetrDetectorError(
|
||||
"Triton native RF-DETR output byte length changed"
|
||||
)
|
||||
return RfDetrRawOutput(boxes=arrays["dets"], logits=arrays["labels"])
|
||||
|
||||
|
||||
def prepare_raw_kb4_rf_detr_native(
|
||||
image_bgr: NDArray[np.uint8],
|
||||
*,
|
||||
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
|
||||
) -> NDArray[np.uint8]:
|
||||
"""Expose the exact raw KB4 raster as UINT8 NHWC without image transforms."""
|
||||
|
||||
if image_bgr.shape != (config.source_height, config.source_width, 3):
|
||||
raise NativeRfDetrDetectorError("raw KB4 image raster changed")
|
||||
if image_bgr.dtype != np.uint8:
|
||||
raise NativeRfDetrDetectorError("raw KB4 image must be uint8")
|
||||
return np.ascontiguousarray(image_bgr[None], dtype=np.uint8)
|
||||
|
||||
|
||||
def postprocess_native_rf_detr(
|
||||
output: RfDetrRawOutput,
|
||||
mask: NDArray[np.bool_],
|
||||
*,
|
||||
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
|
||||
) -> RfDetrPostprocessResult:
|
||||
if output.boxes.shape != (1, 300, 4) or output.logits.shape != (1, 300, 91):
|
||||
raise NativeRfDetrDetectorError("native RF-DETR output shapes are incompatible")
|
||||
if output.boxes.dtype != np.float16 or output.logits.dtype != np.float16:
|
||||
raise NativeRfDetrDetectorError("native RF-DETR output types are incompatible")
|
||||
if not np.isfinite(output.boxes).all() or not np.isfinite(output.logits).all():
|
||||
raise NativeRfDetrDetectorError("native RF-DETR output contains non-finite values")
|
||||
if mask.shape != (config.source_height, config.source_width) or mask.dtype != np.bool_:
|
||||
raise NativeRfDetrDetectorError("valid-FOV mask is incompatible")
|
||||
logits = output.logits[0].astype(np.float32)
|
||||
probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits, -80.0, 80.0)))
|
||||
flattened = probabilities.reshape(-1)
|
||||
topk = np.argsort(-flattened, kind="stable")[: config.maximum_detections]
|
||||
integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1)
|
||||
rejected: Counter[str] = Counter()
|
||||
result: list[RfDetrDetection] = []
|
||||
for flat_index in topk:
|
||||
score = float(flattened[flat_index])
|
||||
if score <= config.minimum_score:
|
||||
continue
|
||||
query_index = int(flat_index // output.logits.shape[2])
|
||||
sparse_class_id = int(flat_index % output.logits.shape[2])
|
||||
class_id = COCO_SPARSE_TO_CONTIGUOUS.get(sparse_class_id)
|
||||
if class_id is None:
|
||||
rejected["unmapped-class-slot"] += 1
|
||||
continue
|
||||
if class_id not in config.target_class_ids:
|
||||
rejected["non-risk-class"] += 1
|
||||
continue
|
||||
center_x, center_y, box_width, box_height = (
|
||||
float(value) for value in output.boxes[0, query_index].astype(np.float32)
|
||||
)
|
||||
box = np.asarray(
|
||||
(
|
||||
(center_x - box_width / 2.0) * config.model_width,
|
||||
(center_y - box_height / 2.0) * config.model_height,
|
||||
(center_x + box_width / 2.0) * config.model_width,
|
||||
(center_y + box_height / 2.0) * config.model_height,
|
||||
),
|
||||
dtype=np.float32,
|
||||
)
|
||||
box[[0, 2]] = np.clip(box[[0, 2]], 0, config.source_width)
|
||||
box[[1, 3]] = np.clip(box[[1, 3]], 0, config.source_height)
|
||||
fraction, center_inside, area = _valid_fraction(box, integral)
|
||||
if area < config.minimum_box_area_pixels:
|
||||
rejected["small-box"] += 1
|
||||
continue
|
||||
if area / (config.source_width * config.source_height) > (
|
||||
config.maximum_box_area_fraction
|
||||
):
|
||||
rejected["large-box"] += 1
|
||||
continue
|
||||
if fraction < config.minimum_valid_fov_fraction:
|
||||
rejected["outside-valid-fov"] += 1
|
||||
continue
|
||||
if config.require_center_inside_valid_fov and not center_inside:
|
||||
rejected["center-outside-valid-fov"] += 1
|
||||
continue
|
||||
result.append(
|
||||
RfDetrDetection(
|
||||
class_id=class_id,
|
||||
label=COCO_CLASSES[class_id],
|
||||
score=round(score, 9),
|
||||
bbox_xyxy=cast(
|
||||
tuple[float, float, float, float],
|
||||
tuple(round(float(value), 6) for value in box),
|
||||
),
|
||||
valid_fov_fraction=round(fraction, 6),
|
||||
)
|
||||
)
|
||||
result.sort(key=lambda item: (-item.score, item.class_id))
|
||||
return RfDetrPostprocessResult(tuple(result), tuple(sorted(rejected.items())))
|
||||
|
||||
|
||||
def _valid_fraction(
|
||||
box: NDArray[np.float32], integral: NDArray[np.int64]
|
||||
) -> tuple[float, bool, float]:
|
||||
height = integral.shape[0] - 1
|
||||
width = integral.shape[1] - 1
|
||||
x1 = int(np.clip(math.floor(float(box[0])), 0, width))
|
||||
y1 = int(np.clip(math.floor(float(box[1])), 0, height))
|
||||
x2 = int(np.clip(math.ceil(float(box[2])), 0, width))
|
||||
y2 = int(np.clip(math.ceil(float(box[3])), 0, height))
|
||||
area = float(max(0, x2 - x1) * max(0, y2 - y1))
|
||||
if area <= 0:
|
||||
return 0.0, False, 0.0
|
||||
inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1]
|
||||
center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1))
|
||||
center_y = int(np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1))
|
||||
center_inside = bool(
|
||||
integral[center_y + 1, center_x + 1]
|
||||
- integral[center_y, center_x + 1]
|
||||
- integral[center_y + 1, center_x]
|
||||
+ integral[center_y, center_x]
|
||||
)
|
||||
return float(inside) / area, center_inside, area
|
||||
|
||||
|
||||
__all__ = [
|
||||
"RF_DETR_NATIVE_CHECKPOINT_SHA256",
|
||||
"RF_DETR_NATIVE_CONFIG",
|
||||
"RF_DETR_NATIVE_CORE_ONNX_SHA256",
|
||||
"RF_DETR_NATIVE_ENGINE_SHA256",
|
||||
"RF_DETR_NATIVE_FP16_ONNX_SHA256",
|
||||
"RF_DETR_NATIVE_MODEL_ID",
|
||||
"RF_DETR_NATIVE_MODEL_VERSION",
|
||||
"RF_DETR_NATIVE_VALID_FOV_SHA256",
|
||||
"RF_DETR_NATIVE_WRAPPED_ONNX_SHA256",
|
||||
"NativeRfDetrConfig",
|
||||
"NativeRfDetrDetectorError",
|
||||
"NativeRfDetrInferenceBackend",
|
||||
"TritonNativeRfDetrHttpInferenceBackend",
|
||||
"postprocess_native_rf_detr",
|
||||
"prepare_raw_kb4_rf_detr_native",
|
||||
]
|
||||
Reference in New Issue
Block a user