SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): add frozen yolox provider

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-05 14:09:51 +03:00
parent 029b486c67
commit a680debfaf
10 changed files with 965 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/perception/__init__.py
+22
View File
@@ -49,6 +49,15 @@ from .contracts import (
TimestampBundle,
validate_exclusive_point_ownership,
)
from .detector import (
FROZEN_YOLOX_MODEL_ID,
FROZEN_YOLOX_PREPROCESS_ID,
FROZEN_YOLOX_PROVIDER_ID,
DetectorProviderError,
DetectorProviderSnapshot,
FrozenYoloxDetectorProvider,
proposals_from_detections,
)
from .graph import (
GRAPH_RESULT_SCHEMA,
REFERENCE_GRAPH_ID,
@@ -78,8 +87,11 @@ from .providers import (
ThreatProvider,
)
from .recorded_source import (
DecodedRecordedSource,
LiveSourceAdapter,
PyAvRecordedImageDecoder,
RecordedFrameReference,
RecordedImageDecoder,
RecordedRavnoves00Source,
RecordedSourceError,
ReplayPacing,
@@ -129,6 +141,13 @@ __all__ = [
"ThreatDecision",
"TimestampBundle",
"validate_exclusive_point_ownership",
"FROZEN_YOLOX_MODEL_ID",
"FROZEN_YOLOX_PREPROCESS_ID",
"FROZEN_YOLOX_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"FrozenYoloxDetectorProvider",
"proposals_from_detections",
"REFERENCE_GRAPH_CONFIG_SCHEMA",
"DetectorProvider",
"GeometryAssociationProvider",
@@ -154,7 +173,10 @@ __all__ = [
"TerminalOutcome",
"TerminalOutcomeType",
"LiveSourceAdapter",
"DecodedRecordedSource",
"PyAvRecordedImageDecoder",
"RecordedFrameReference",
"RecordedImageDecoder",
"RecordedRavnoves00Source",
"RecordedSourceError",
"ReplayPacing",
src/k1link/perception/baseline.py
+5 -1
View File
@@ -21,6 +21,9 @@ BASELINE_SOURCE_ID: Final = "RAVNOVES00"
BASELINE_SESSION_ID: Final = "20260720T065719Z_viewer_live"
BASELINE_CAMERA_SOURCE_ID: Final = "sensor.camera.right"
BASELINE_RECORDED_JOB_ID: Final = "recorded-camera-602ac89026ed12978619801d"
BASELINE_CAMERA_STREAM_SHA256: Final = (
"cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8"
)
BASELINE_SOURCE_PACK_ID: Final = (
"e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b"
)
@@ -124,7 +127,8 @@ def load_m4_baseline(path: Path) -> BaselineProfile:
raise BaselineContractError("M4 source pack identity changed")
if source.get("source_pack_artifact_sha256") != BASELINE_SOURCE_PACK_SHA256:
raise BaselineContractError("M4 source pack artifact identity changed")
_digest(source.get("camera_stream_sha256"), "camera stream digest")
if source.get("camera_stream_sha256") != BASELINE_CAMERA_STREAM_SHA256:
raise BaselineContractError("M4 camera stream identity changed")
modalities = _string_array(source.get("modalities"), "source modalities")
if set(modalities) != {"image", "registered-point-increment", "pose"}:
raise BaselineContractError("baseline source must bind image, points and pose")
src/k1link/perception/detector.py
+156
View File
@@ -0,0 +1,156 @@
"""Frozen raw-KB4 YOLOX provider for class-agnostic object proposals."""
from __future__ import annotations
import time
from collections import Counter
from collections.abc import Callable
from dataclasses import dataclass
from threading import Lock
from typing import Final
import numpy as np
from numpy.typing import NDArray
from k1link.compute.yolox_object_detector import (
FROZEN_YOLOX_CONFIG,
YOLOX_MODEL_ID,
YOLOX_MODEL_VERSION,
FrozenYoloxConfig,
ImageResizer,
InferenceBackend,
YoloxDetection,
postprocess_yolox,
preprocess_raw_kb4,
)
from .contracts import BoundingRegion2D, ObjectProposal2D
from .providers import SourcePacket
FROZEN_YOLOX_PROVIDER_ID: Final = "triton-yolox-s-raw-kb4/v1"
FROZEN_YOLOX_MODEL_ID: Final = f"{YOLOX_MODEL_ID}:{YOLOX_MODEL_VERSION}"
FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
class DetectorProviderError(RuntimeError):
"""The detector input, frozen inference or proposal output is incompatible."""
@dataclass(frozen=True, slots=True)
class DetectorProviderSnapshot:
input_frames: int
completed_frames: int
failed_frames: int
zero_proposal_frames: int
proposal_count: int
rejected: tuple[tuple[str, int], ...]
core_duration_ns: int
class FrozenYoloxDetectorProvider:
"""One image payload produces one frozen inference request and proposal tuple."""
provider_id: str = FROZEN_YOLOX_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: InferenceBackend,
resizer: ImageResizer | None = None,
config: FrozenYoloxConfig = FROZEN_YOLOX_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
raise DetectorProviderError("frozen valid-FOV mask is incompatible")
self.mask = np.asarray(mask, dtype=np.bool_)
self.backend = backend
self.resizer = resizer
self.config = config
self._clock_ns = clock_ns
self._lock = Lock()
self._input_frames = 0
self._completed_frames = 0
self._failed_frames = 0
self._zero_proposal_frames = 0
self._proposal_count = 0
self._rejected: Counter[str] = Counter()
self._core_duration_ns = 0
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
payload = packet.image_payload
with self._lock:
self._input_frames += 1
started_ns = int(self._clock_ns())
try:
if not isinstance(payload, np.ndarray):
raise DetectorProviderError("detector requires a decoded BGR image payload")
image = np.asarray(payload)
if image.dtype != np.uint8:
raise DetectorProviderError("decoded BGR image must be uint8")
tensor = preprocess_raw_kb4(
image,
self.mask,
config=self.config,
resizer=self.resizer,
)
output = self.backend.infer(tensor)
postprocessed = postprocess_yolox(output, self.mask, config=self.config)
proposals = proposals_from_detections(packet, postprocessed.detections)
except Exception:
with self._lock:
self._failed_frames += 1
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
raise
with self._lock:
self._completed_frames += 1
self._proposal_count += len(proposals)
self._zero_proposal_frames += not proposals
self._rejected.update(dict(postprocessed.rejected))
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
return proposals
def snapshot(self) -> DetectorProviderSnapshot:
with self._lock:
return DetectorProviderSnapshot(
input_frames=self._input_frames,
completed_frames=self._completed_frames,
failed_frames=self._failed_frames,
zero_proposal_frames=self._zero_proposal_frames,
proposal_count=self._proposal_count,
rejected=tuple(sorted(self._rejected.items())),
core_duration_ns=self._core_duration_ns,
)
def proposals_from_detections(
packet: SourcePacket,
detections: tuple[YoloxDetection, ...],
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
ObjectProposal2D(
proposal_id=f"proposal-{envelope.sequence}-{index}",
source_id=envelope.source_id,
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=FROZEN_YOLOX_PROVIDER_ID,
model_id=FROZEN_YOLOX_MODEL_ID,
preprocess_id=FROZEN_YOLOX_PREPROCESS_ID,
semantic_hint=detection.label,
provider_tracklet=None,
)
for index, detection in enumerate(detections)
)
__all__ = [
"FROZEN_YOLOX_MODEL_ID",
"FROZEN_YOLOX_PREPROCESS_ID",
"FROZEN_YOLOX_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"FrozenYoloxDetectorProvider",
"proposals_from_detections",
]
src/k1link/perception/recorded_source.py
+78 -2
View File
@@ -3,18 +3,21 @@
from __future__ import annotations
import hashlib
import importlib
import json
import time
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from dataclasses import dataclass, replace
from enum import StrEnum
from pathlib import Path
from threading import Event
from typing import Final, Protocol
from typing import Any, Final, Protocol, cast
import numpy as np
from numpy.typing import NDArray
from .baseline import (
BASELINE_CAMERA_STREAM_SHA256,
BASELINE_PROFILE_ID,
BASELINE_RECORDED_JOB_ID,
BASELINE_SESSION_ID,
@@ -75,6 +78,10 @@ class LiveSourceAdapter(Protocol):
def packets(self, stop_event: Event) -> Iterator[SourcePacket]: ...
class RecordedImageDecoder(Protocol):
def frames(self, stop_event: Event) -> Iterator[NDArray[np.uint8]]: ...
WaitFunction = Callable[[Event, float], bool]
@@ -165,6 +172,72 @@ class RecordedRavnoves00Source:
return False
class DecodedRecordedSource:
"""Attach decoded BGR frames without introducing detector logic into the source."""
provider_id: str = RECORDED_SOURCE_PROVIDER_ID
def __init__(
self,
*,
source: RecordedRavnoves00Source,
decoder: RecordedImageDecoder,
) -> None:
self.source = source
self.decoder = decoder
def packets(self, stop_event: Event) -> Iterator[SourcePacket]:
images = self.decoder.frames(stop_event)
for packet in self.source.packets(stop_event):
try:
image = next(images)
except StopIteration as exc:
raise RecordedSourceError(
"decoded image stream ended before source timeline"
) from exc
if image.shape != (600, 800, 3) or image.dtype != np.uint8:
raise RecordedSourceError("decoded image raster is incompatible")
yield replace(packet, image_payload=image)
if not stop_event.is_set():
try:
next(images)
except StopIteration:
return
raise RecordedSourceError("decoded image stream exceeds source timeline")
class PyAvRecordedImageDecoder:
"""Sequential full-video decoder used by the Worker 006 recorded source adapter."""
def __init__(
self,
path: Path,
*,
expected_sha256: str = BASELINE_CAMERA_STREAM_SHA256,
) -> None:
self.path = path.resolve(strict=True)
if self.path.is_symlink() or _file_sha256(self.path) != expected_sha256:
raise RecordedSourceError("recorded camera video identity changed")
def frames(self, stop_event: Event) -> Iterator[NDArray[np.uint8]]:
try:
av: Any = importlib.import_module("av")
except ModuleNotFoundError as exc:
raise RecordedSourceError("PyAV decoder is unavailable") from exc
container: Any = av.open(str(self.path))
try:
streams = container.streams.video
if len(streams) != 1:
raise RecordedSourceError("recorded camera video stream count changed")
for decoded in container.decode(streams[0]):
if stop_event.is_set():
return
image = cast(NDArray[np.uint8], decoded.to_ndarray(format="bgr24"))
yield np.asarray(image, dtype=np.uint8)
finally:
container.close()
def _packet(
frame_index: int,
camera: dict[str, object],
@@ -316,8 +389,11 @@ def _event_wait(stop_event: Event, timeout_seconds: float) -> bool:
__all__ = [
"BASELINE_PROFILE_ID",
"LiveSourceAdapter",
"DecodedRecordedSource",
"PyAvRecordedImageDecoder",
"RECORDED_SOURCE_PROVIDER_ID",
"RecordedFrameReference",
"RecordedImageDecoder",
"RecordedRavnoves00Source",
"RecordedSourceError",
"ReplayPacing",