SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): evaluate fixed-class detector candidates

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 16:44:15 +03:00
parent 6276bbf324
commit 33cef2fdea
28 changed files with 4498 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/perception/detector.py
+154 -4
View File
@@ -1,4 +1,4 @@
"""Frozen raw-KB4 YOLOX provider for class-agnostic object proposals."""
"""Versioned fixed-class detector providers for raw-KB4 object proposals."""
from __future__ import annotations
@@ -14,10 +14,22 @@ from numpy.typing import NDArray
from .contracts import BoundingRegion2D, ObjectProposal2D
from .providers import SourcePacket
from .rf_detr_object_detector import (
RF_DETR_CONFIG,
RF_DETR_MODEL_ID,
RF_DETR_MODEL_VERSION,
RfDetrConfig,
RfDetrDetection,
RfDetrInferenceBackend,
postprocess_rf_detr,
preprocess_raw_kb4_rf_detr,
)
from .yolox_object_detector import (
ALL_COCO_YOLOX_CONFIG,
FROZEN_YOLOX_CONFIG,
YOLOX_MODEL_ID,
YOLOX_MODEL_VERSION,
AllCocoYoloxConfig,
FrozenYoloxConfig,
ImageResizer,
InferenceBackend,
@@ -27,8 +39,12 @@ from .yolox_object_detector import (
)
FROZEN_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4/v1"
ALL_COCO_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4-all-coco/v2"
FROZEN_YOLOX_MODEL_ID: Final = f"{YOLOX_MODEL_ID}:{YOLOX_MODEL_VERSION}"
FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}"
RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0"
class DetectorProviderError(RuntimeError):
@@ -57,7 +73,7 @@ class FrozenYoloxDetectorProvider:
mask: NDArray[np.bool_],
backend: InferenceBackend,
resizer: ImageResizer | None = None,
config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG,
config: FrozenYoloxConfig | AllCocoYoloxConfig = FROZEN_YOLOX_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
@@ -95,7 +111,11 @@ class FrozenYoloxDetectorProvider:
)
output = self.backend.infer(tensor)
postprocessed = postprocess_yolox(output, self.mask, config=self.config)
proposals = proposals_from_detections(packet, postprocessed.detections)
proposals = proposals_from_detections(
packet,
postprocessed.detections,
provider_id=self.provider_id,
)
except Exception:
with self._lock:
self._failed_frames += 1
@@ -125,6 +145,8 @@ class FrozenYoloxDetectorProvider:
def proposals_from_detections(
packet: SourcePacket,
detections: tuple[YoloxDetection, ...],
*,
provider_id: str = FROZEN_YOLOX_PROVIDER_ID,
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
@@ -134,7 +156,7 @@ def proposals_from_detections(
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=FROZEN_YOLOX_PROVIDER_ID,
provider_id=provider_id,
model_id=FROZEN_YOLOX_MODEL_ID,
preprocess_id=FROZEN_YOLOX_PREPROCESS_ID,
semantic_hint=detection.label,
@@ -144,12 +166,140 @@ def proposals_from_detections(
)
class AllCocoYoloxDetectorProvider(FrozenYoloxDetectorProvider):
"""Emit every qualified COCO class without adding another inference pass."""
provider_id: str = ALL_COCO_YOLOX_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: InferenceBackend,
resizer: ImageResizer | None = None,
config: AllCocoYoloxConfig = ALL_COCO_YOLOX_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
super().__init__(
mask=mask,
backend=backend,
resizer=resizer,
config=config,
clock_ns=clock_ns,
)
class RfDetrShadowDetectorProvider:
"""Emit behavior-relevant fixed classes from one RF-DETR inference pass."""
provider_id: str = RF_DETR_SHADOW_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: RfDetrInferenceBackend,
resizer: ImageResizer | None = None,
config: RfDetrConfig = RF_DETR_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
raise DetectorProviderError("RF-DETR valid-FOV mask is incompatible")
self.mask = np.asarray(mask, dtype=np.bool_)
self.backend = backend
self.resizer = resizer
self.config = config
self._clock_ns = clock_ns
self._lock = Lock()
self._input_frames = 0
self._completed_frames = 0
self._failed_frames = 0
self._zero_proposal_frames = 0
self._proposal_count = 0
self._rejected: Counter[str] = Counter()
self._core_duration_ns = 0
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
payload = packet.image_payload
with self._lock:
self._input_frames += 1
started_ns = int(self._clock_ns())
try:
if not isinstance(payload, np.ndarray):
raise DetectorProviderError("RF-DETR requires a decoded BGR image payload")
image = np.asarray(payload)
if image.dtype != np.uint8:
raise DetectorProviderError("decoded BGR image must be uint8")
tensor = preprocess_raw_kb4_rf_detr(
image,
self.mask,
config=self.config,
resizer=self.resizer,
)
output = self.backend.infer(tensor)
postprocessed = postprocess_rf_detr(output, self.mask, config=self.config)
proposals = proposals_from_rf_detr_detections(packet, postprocessed.detections)
except Exception:
with self._lock:
self._failed_frames += 1
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
raise
with self._lock:
self._completed_frames += 1
self._proposal_count += len(proposals)
self._zero_proposal_frames += not proposals
self._rejected.update(dict(postprocessed.rejected))
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
return proposals
def snapshot(self) -> DetectorProviderSnapshot:
with self._lock:
return DetectorProviderSnapshot(
input_frames=self._input_frames,
completed_frames=self._completed_frames,
failed_frames=self._failed_frames,
zero_proposal_frames=self._zero_proposal_frames,
proposal_count=self._proposal_count,
rejected=tuple(sorted(self._rejected.items())),
core_duration_ns=self._core_duration_ns,
)
def proposals_from_rf_detr_detections(
packet: SourcePacket,
detections: tuple[RfDetrDetection, ...],
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
ObjectProposal2D(
proposal_id=f"proposal-{envelope.sequence}-{index}",
source_id=envelope.source_id,
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=RF_DETR_SHADOW_PROVIDER_ID,
model_id=RF_DETR_SHADOW_MODEL_ID,
preprocess_id=RF_DETR_SHADOW_PREPROCESS_ID,
semantic_hint=detection.label,
provider_tracklet=None,
)
for index, detection in enumerate(detections)
)
__all__ = [
"ALL_COCO_YOLOX_PROVIDER_ID",
"FROZEN_YOLOX_MODEL_ID",
"FROZEN_YOLOX_PREPROCESS_ID",
"FROZEN_YOLOX_PROVIDER_ID",
"RF_DETR_SHADOW_MODEL_ID",
"RF_DETR_SHADOW_PREPROCESS_ID",
"RF_DETR_SHADOW_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"AllCocoYoloxDetectorProvider",
"FrozenYoloxDetectorProvider",
"RfDetrShadowDetectorProvider",
"proposals_from_detections",
"proposals_from_rf_detr_detections",
]