SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): integrate native RF-DETR shadow provider

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-26 02:06:16 +03:00
parent 28effdde23
commit b111406cf8
9 changed files with 1489 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/perception/detector.py
+181
View File
@@ -14,6 +14,15 @@ from numpy.typing import NDArray
from .contracts import BoundingRegion2D, ObjectProposal2D
from .providers import SourcePacket
from .rf_detr_native_object_detector import (
RF_DETR_NATIVE_CONFIG,
RF_DETR_NATIVE_MODEL_ID,
RF_DETR_NATIVE_MODEL_VERSION,
NativeRfDetrConfig,
NativeRfDetrInferenceBackend,
postprocess_native_rf_detr,
prepare_raw_kb4_rf_detr_native,
)
from .rf_detr_object_detector import (
RF_DETR_CONFIG,
RF_DETR_MODEL_ID,
@@ -45,6 +54,15 @@ FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}"
RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0"
RF_DETR_NATIVE_SHADOW_PROVIDER_ID: Final = (
"triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0"
)
RF_DETR_NATIVE_SHADOW_MODEL_ID: Final = (
f"{RF_DETR_NATIVE_MODEL_ID}:{RF_DETR_NATIVE_MODEL_VERSION}"
)
RF_DETR_NATIVE_SHADOW_PREPROCESS_ID: Final = (
"raw-kb4-uint8-fused-mask-rgb-pad8-imagenet-trt/v0"
)
class DetectorProviderError(RuntimeError):
@@ -414,6 +432,164 @@ def proposals_from_rf_detr_detections(
)
class NativeRfDetrShadowDetectorProvider:
"""Emit risk classes from one exact-raster native RF-DETR inference pass."""
provider_id: str = RF_DETR_NATIVE_SHADOW_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: NativeRfDetrInferenceBackend,
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
timing_observer: DetectorTimingObserver | None = None,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
raise DetectorProviderError("native RF-DETR valid-FOV mask is incompatible")
self.mask = np.asarray(mask, dtype=np.bool_)
self.backend = backend
self.config = config
self._clock_ns = clock_ns
self.timing_observer = timing_observer
self._lock = Lock()
self._input_frames = 0
self._completed_frames = 0
self._failed_frames = 0
self._zero_proposal_frames = 0
self._proposal_count = 0
self._rejected: Counter[str] = Counter()
self._core_duration_ns = 0
self._warmup_started = False
self._warmup_snapshot: DetectorWarmupSnapshot | None = None
def warm_up(self) -> DetectorWarmupSnapshot:
"""Prime raw transport and postprocessing before source admission."""
with self._lock:
if self._warmup_snapshot is not None:
return self._warmup_snapshot
if self._warmup_started:
raise DetectorProviderError("native RF-DETR warmup is already in progress")
self._warmup_started = True
started_ns = int(self._clock_ns())
try:
image = np.zeros(
(self.config.source_height, self.config.source_width, 3),
dtype=np.uint8,
)
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
preprocessed_ns = int(self._clock_ns())
output = self.backend.infer(tensor)
inferred_ns = int(self._clock_ns())
postprocess_native_rf_detr(output, self.mask, config=self.config)
completed_ns = int(self._clock_ns())
except Exception:
with self._lock:
self._warmup_started = False
raise
snapshot = DetectorWarmupSnapshot(
completed=True,
inference_passes=1,
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
total_duration_ns=max(0, completed_ns - started_ns),
)
with self._lock:
self._warmup_snapshot = snapshot
return snapshot
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
payload = packet.image_payload
with self._lock:
self._input_frames += 1
started_ns = int(self._clock_ns())
try:
if not isinstance(payload, np.ndarray):
raise DetectorProviderError(
"native RF-DETR requires a decoded BGR image payload"
)
image = np.asarray(payload)
if image.dtype != np.uint8:
raise DetectorProviderError("decoded BGR image must be uint8")
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
preprocessed_ns = (
int(self._clock_ns()) if self.timing_observer is not None else started_ns
)
output = self.backend.infer(tensor)
inferred_ns = (
int(self._clock_ns()) if self.timing_observer is not None else preprocessed_ns
)
postprocessed = postprocess_native_rf_detr(
output,
self.mask,
config=self.config,
)
proposals = proposals_from_native_rf_detr_detections(
packet,
postprocessed.detections,
)
except Exception:
with self._lock:
self._failed_frames += 1
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
raise
completed_ns = int(self._clock_ns())
with self._lock:
self._completed_frames += 1
self._proposal_count += len(proposals)
self._zero_proposal_frames += not proposals
self._rejected.update(dict(postprocessed.rejected))
self._core_duration_ns += max(0, completed_ns - started_ns)
if self.timing_observer is not None:
self.timing_observer(
DetectorFrameTiming(
sequence=packet.envelope.sequence,
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
total_duration_ns=max(0, completed_ns - started_ns),
)
)
return proposals
def snapshot(self) -> DetectorProviderSnapshot:
with self._lock:
return DetectorProviderSnapshot(
input_frames=self._input_frames,
completed_frames=self._completed_frames,
failed_frames=self._failed_frames,
zero_proposal_frames=self._zero_proposal_frames,
proposal_count=self._proposal_count,
rejected=tuple(sorted(self._rejected.items())),
core_duration_ns=self._core_duration_ns,
)
def proposals_from_native_rf_detr_detections(
packet: SourcePacket,
detections: tuple[RfDetrDetection, ...],
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
ObjectProposal2D(
proposal_id=f"proposal-{envelope.sequence}-{index}",
source_id=envelope.source_id,
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
model_id=RF_DETR_NATIVE_SHADOW_MODEL_ID,
preprocess_id=RF_DETR_NATIVE_SHADOW_PREPROCESS_ID,
semantic_hint=detection.label,
provider_tracklet=None,
)
for index, detection in enumerate(detections)
)
__all__ = [
"ALL_COCO_YOLOX_PROVIDER_ID",
"FROZEN_YOLOX_MODEL_ID",
@@ -422,6 +598,9 @@ __all__ = [
"RF_DETR_SHADOW_MODEL_ID",
"RF_DETR_SHADOW_PREPROCESS_ID",
"RF_DETR_SHADOW_PROVIDER_ID",
"RF_DETR_NATIVE_SHADOW_MODEL_ID",
"RF_DETR_NATIVE_SHADOW_PREPROCESS_ID",
"RF_DETR_NATIVE_SHADOW_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"DetectorFrameTiming",
@@ -429,7 +608,9 @@ __all__ = [
"DetectorWarmupSnapshot",
"AllCocoYoloxDetectorProvider",
"FrozenYoloxDetectorProvider",
"NativeRfDetrShadowDetectorProvider",
"RfDetrShadowDetectorProvider",
"proposals_from_detections",
"proposals_from_native_rf_detr_detections",
"proposals_from_rf_detr_detections",
]