SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): integrate native RF-DETR shadow provider

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-26 02:06:16 +03:00
parent 28effdde23
commit b111406cf8
9 changed files with 1489 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json
+95
View File
@@ -0,0 +1,95 @@
{
"schema_version": "missioncore.reference-perception-graph-config/v2",
"graph_id": "reference-perception-graph/v2",
"source_profile_id": "m4-ravnoves00-recorded-realtime/v1",
"providers": [
{
"role": "source",
"provider_id": "ravnoves00-recorded-source/v1",
"version": "1.0.0",
"revision": "m4-ravnoves00-recorded-realtime/v1",
"sha256": "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
},
{
"role": "detector",
"provider_id": "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0",
"version": "0.1.0",
"revision": "rf-detr-large-coco-native-kb4-uint8-trt11-fp16-risk-shadow/v0",
"sha256": "398b1102943e704b08a033b1d04b6bdb039ecdd73a2b2e370a0a9cbcaff501ec"
},
{
"role": "geometry",
"provider_id": "ravnoves00-geometry-association/v1",
"version": "1.0.0",
"revision": "m4-ravnoves00-e29-e32-geometry/v1",
"sha256": "cc666c9389a5e221957faddec89584709b66918d14abaf646f1832e001421999"
},
{
"role": "temporal",
"provider_id": "bounded-spatial-temporal-layer/v1",
"version": "1.0.0",
"revision": "m4-bounded-temporal-motion/v1",
"sha256": "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39"
},
{
"role": "motion",
"provider_id": "class-independent-motion-estimator/v1",
"version": "1.0.0",
"revision": "m4-bounded-temporal-motion/v1",
"sha256": "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39"
},
{
"role": "rolling",
"provider_id": "rolling-local-obstacle-map/v1",
"version": "1.0.0",
"revision": "ravnoves00-rolling-local-obstacle-map/v1",
"sha256": "f7e3315eaf6ffaf3aee1e04913933812092cf82bbcc9984c1a6fa2d9250e6784"
},
{
"role": "threat",
"provider_id": "dual-evidence-replay-threat/v3",
"version": "3.0.0",
"revision": "m4-ravnoves00-virtual-corridor/v3",
"sha256": "8c3a5aa837da1f028f5998fb504a1381f9b2b68de6420a32160410b6dc0887c7"
}
],
"queues": [
{
"stage_id": "detector",
"capacity": 2,
"deadline_ns": 1000000000,
"terminal_timeout_ns": 90000000000
},
{
"stage_id": "geometry",
"capacity": 2,
"deadline_ns": 1500000000,
"terminal_timeout_ns": 90000000000
},
{
"stage_id": "temporal",
"capacity": 2,
"deadline_ns": 1750000000,
"terminal_timeout_ns": 90000000000
},
{
"stage_id": "rolling",
"capacity": 2,
"deadline_ns": 2000000000,
"terminal_timeout_ns": 90000000000
},
{
"stage_id": "threat",
"capacity": 2,
"deadline_ns": 2250000000,
"terminal_timeout_ns": 90000000000
}
],
"authority": {
"mode": "replay-simulated",
"physical_live": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json
+114
View File
@@ -0,0 +1,114 @@
{
"schema_version": "missioncore.rf-detr-native-risk-shadow-profile/v0",
"profile_id": "rf-detr-large-coco-native-kb4-uint8-trt11-fp16-risk-shadow/v0",
"provider_id": "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0",
"model": {
"model_id": "rf_detr_large_native_kb4",
"model_version": 1,
"upstream_version": "1.9.4",
"upstream_revision": "9b009fa928d6218320439803d1da01869a85c072",
"checkpoint_sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38",
"native_core_onnx_sha256": "62e549748a1d17646b90ad06d3ac8a1b79595b7e9270cac4564418f023079176",
"strongly_typed_fp16_onnx_sha256": "00b29fa2ff3d5fca730ebf3b8c33b10e9d690cc97bbbbfa5563d6e0abf210999",
"fused_uint8_onnx_sha256": "acdd01623a00d100331473c0a99eab1e5adf33117cbab900c8ae078edd4aa346",
"worker_006_rtx4090_tensorrt_11_engine_sha256": "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695",
"input": {
"name": "raw_kb4_bgr",
"datatype": "UINT8",
"shape": [1, 600, 800, 3],
"bytes_per_frame": 1440000
},
"outputs": [
{"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]},
{"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]}
]
},
"preprocessing": {
"execution": "single-fused-tensorrt-gpu-graph",
"source_raster": [800, 600],
"source_color": "BGR",
"model_canvas": [800, 608],
"model_color": "RGB",
"valid_fov_mask_sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63",
"valid_fov_fill_value": 114,
"padding_tblr": [0, 8, 0, 0],
"normalization_mean": [0.485, 0.456, 0.406],
"normalization_std": [0.229, 0.224, 0.225],
"resize": false,
"crop": false,
"rectification": false,
"warp": false,
"geometric_resampling": false
},
"emission": {
"single_inference_per_source_frame": true,
"minimum_score": 0.25,
"maximum_topk_query_class_pairs": 300,
"behavior_relevant_classes": [
"person",
"bicycle",
"car",
"motorcycle",
"bus",
"truck",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"skateboard"
],
"geometry_owns_static_occupancy": true,
"unlisted_semantic_classes_emitted": false,
"minimum_box_area_pixels": 64,
"maximum_box_area_fraction": 0.5,
"minimum_valid_fov_fraction": 0.5,
"require_center_inside_valid_fov": true
},
"qualification": {
"native_pytorch_tensorrt_parity": {
"report_identity_sha256": "215145fed04b43670a71594a7ea6d8f2a676f2ee781eb1d0bfc05bb3ecdbf17c",
"passed": true,
"risk_detection_precision": 0.988700565,
"risk_detection_recall": 0.983146067,
"matched_mean_iou": 0.988242066
},
"full_ravnoves00_native_vs_legacy_704": {
"report_identity_sha256": "729bf02b5b52b22d347b3c960f3ac01539ebc76569273bd85560fed8d7b7616b",
"frame_count": 4489,
"native_total_mean_ms": 12.334212,
"native_total_p95_ms": 19.730654,
"legacy_704_total_mean_ms": 33.995721,
"legacy_704_total_p95_ms": 47.432686,
"transport_bytes_reduction_fraction": 0.757877066,
"native_detection_count": 48583,
"legacy_704_detection_count": 51691,
"legacy_box_agreement_recall_iou_at_least_0_5": 0.766593798,
"legacy_box_agreement_gate_passed": false,
"interpretation": "diagnostic-only because legacy 704 geometrically stretches the raw 4:3 raster"
}
},
"queue": {
"policy": "bounded-latest-wins",
"capacity": 2
},
"status": {
"native_tensor_parity_passed": true,
"full_ravnoves00_runtime_gate_passed": true,
"legacy_704_box_agreement_gate_passed": false,
"integrated_world_state_gate_passed": false,
"production_accepted": false
},
"authority": {
"ground_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
experiments/perception/run_m48s_reference_graph_shadow_worker.py
+12 -2
View File
@@ -25,6 +25,7 @@ from k1link.perception.detector import (
DetectorFrameTiming,
DetectorProviderSnapshot,
DetectorWarmupSnapshot,
NativeRfDetrShadowDetectorProvider,
RfDetrShadowDetectorProvider,
)
from k1link.perception.geometry import Ravnoves00GeometryAssociationProvider
@@ -446,6 +447,7 @@ def main() -> int:
all_pipeline_timings: list[dict[str, object]] = []
all_decode_timings: list[DecodedFrameTiming] = []
all_pacing_timings: list[SourcePacingTiming] = []
detector_provider_id: str | None = None
with (
progress.open("x", encoding="utf-8") as progress_stream,
frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream,
@@ -478,6 +480,11 @@ def main() -> int:
maximum_frames=arguments.maximum_frames,
source_rate_hz=arguments.source_rate_hz,
) as runtime:
current_detector_provider_id = runtime.graph.detector.provider_id
if detector_provider_id is None:
detector_provider_id = current_detector_provider_id
elif detector_provider_id != current_detector_provider_id:
raise RuntimeError("detector provider identity changed between loops")
for stage_id, attribute in (
("geometry", "geometry"),
("temporal", "temporal"),
@@ -504,7 +511,8 @@ def main() -> int:
result = runtime.graph.run()
loop_completed_ns = time.monotonic_ns()
detector_snapshot = cast(
RfDetrShadowDetectorProvider,
RfDetrShadowDetectorProvider
| NativeRfDetrShadowDetectorProvider,
runtime.graph.detector,
).snapshot()
provider_snapshots = {
@@ -582,6 +590,8 @@ def main() -> int:
print(json.dumps(progress_row, sort_keys=True), flush=True)
completed_ns = time.monotonic_ns()
if detector_provider_id is None:
raise RuntimeError("detector provider identity was not observed")
wall_seconds = (completed_ns - started_ns) / 1_000_000_000.0
rss_after_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
accounting: Counter[str] = Counter()
@@ -674,7 +684,7 @@ def main() -> int:
"identity": {
"worker_id": "worker-006",
"graph_id": "reference-perception-graph/v2",
"detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0",
"detector_provider_id": detector_provider_id,
"inputs": _input_digests(paths, arguments.detector_profile),
"runtime_artifact_sha256": arguments.runtime_artifact_sha256,
"runner_sha256": arguments.runner_sha256,
experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1
+360
View File
@@ -0,0 +1,360 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$ReleaseRoot,
[Parameter(Mandatory = $true)]
[string]$CandidateRoot,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedWheelSha256,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9._-]{1,96}$")]
[string]$RunId,
[ValidateRange(1, 4489)]
[int]$MaximumFrames = 300,
[ValidateRange(1.0, 120.0)]
[double]$SourceRateHz = 10.0,
[ValidateRange(0.0, 1.0)]
[double]$MinimumDeliveryRatio = 0.999,
[ValidateRange(0.1, 120.0)]
[double]$MinimumEffectiveWorldStateFps = 9.5,
[ValidateRange(1.0, 10000.0)]
[double]$MaximumWorldStateCompletionP95Ms = 125.0,
[string]$OutputRoot = (
"D:\NDC_MISSIONCORE\runtime\results\m48n-native-reference-graph-shadow"
)
)
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
function Assert-LastExitCode([string]$Operation) {
if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" }
}
function Get-Sha256([string]$Path) {
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Assert-File([string]$Path, [string]$ExpectedSha256, [string]$Label) {
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw "$Label must be a regular file"
}
if ((Get-Sha256 $item.FullName) -cne $ExpectedSha256) {
throw "$Label SHA-256 changed"
}
return $item.FullName
}
function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) {
if ($Create -and -not (Test-Path -LiteralPath $Path)) {
$null = New-Item -ItemType Directory -Path $Path
}
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
if (
-not $item.PSIsContainer -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
[IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:"
) {
throw "$Label must be a real D: directory"
}
return $item.FullName
}
function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") }
function Get-Container([string]$Name) {
$rows = @((& docker inspect $Name) | ConvertFrom-Json)
Assert-LastExitCode "Docker inspection for $Name"
if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" }
return $rows[0]
}
if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") {
throw "M48N native reference graph is pinned to DESKTOP-OPJ8J04"
}
$release = Resolve-DDirectory $ReleaseRoot "M48N release root" $false
$candidate = Resolve-DDirectory $CandidateRoot "M48N candidate root" $false
$output = Resolve-DDirectory $OutputRoot "M48N output root" $true
$runOutput = Join-Path $output $RunId
if (Test-Path -LiteralPath $runOutput) { throw "M48N run output already exists" }
$null = New-Item -ItemType Directory -Path $runOutput
$runOutput = Resolve-DDirectory $runOutput "M48N run output" $false
$wheel = Assert-File (
Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl"
) $ExpectedWheelSha256 "M48N wheel"
$expectedConfigs = [ordered]@{
"m48n-rf-detr-native-reference-graph-shadow-v0.json" = (
"336dccb6b64f4fa5def14aae967fd3280cc1720e93fa3ee21885e0c3304cee4d"
)
"m4-recorded-realtime-baseline-v1.json" = (
"ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8"
)
"rf-detr-large-native-kb4-risk-shadow-v0.json" = (
"398b1102943e704b08a033b1d04b6bdb039ecdd73a2b2e370a0a9cbcaff501ec"
)
"m4-geometry-association-v1.json" = (
"cc666c9389a5e221957faddec89584709b66918d14abaf646f1832e001421999"
)
"m4-temporal-motion-v1.json" = (
"7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39"
)
"m4-rolling-local-map-v1.json" = (
"f7e3315eaf6ffaf3aee1e04913933812092cf82bbcc9984c1a6fa2d9250e6784"
)
"m4-replay-threat-v3.json" = (
"8c3a5aa837da1f028f5998fb504a1381f9b2b68de6420a32160410b6dc0887c7"
)
}
foreach ($entry in $expectedConfigs.GetEnumerator()) {
$null = Assert-File (Join-Path $release $entry.Key) $entry.Value (
"M48N config {0}" -f $entry.Key
)
}
$runner = Get-Item -LiteralPath (
Join-Path $release "run_m48s_reference_graph_shadow_worker.py"
)
if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) {
throw "M48N graph runner must be a regular file"
}
$runnerSha256 = Get-Sha256 $runner.FullName
$nativeConfig = Assert-File (
Join-Path $release "rf_detr_large_native_kb4_config.pbtxt"
) "15e100029df92c1390c567517eac6d8bf640591c865bf87a3955289292ba3a22" (
"native RF-DETR Triton config"
)
$nativeEngine = Assert-File (
Join-Path $candidate "rf-detr-native-uint8.plan"
) "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695" (
"native RF-DETR TensorRT engine"
)
$modelRoot = Join-Path $runOutput "triton-models"
$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4"
$modelVersionDirectory = Join-Path $modelDirectory "1"
$null = New-Item -ItemType Directory -Path $modelVersionDirectory
Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $modelDirectory "config.pbtxt")
Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $modelVersionDirectory "model.plan")
if (
(Get-Sha256 (Join-Path $modelVersionDirectory "model.plan")) -cne
"b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695"
) {
throw "staged native RF-DETR TensorRT engine SHA-256 changed"
}
$source = [ordered]@{
CameraIndex = (
"D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d" +
"\input\camera\sensor.camera.right\epoch-1\index.jsonl"
)
SourcePack = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" +
"\lidar-pack.npz"
)
LocalSurface = (
"D:\NDC_MISSIONCORE\runtime\derived" +
"\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55" +
"\local-surface.npz"
)
Video = (
"D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" +
"\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
)
Mask = (
"D:\NDC_MISSIONCORE\runtime\inputs\e2" +
"\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +
"\mask.png"
)
}
foreach ($entry in $source.GetEnumerator()) {
if (-not (Test-Path -LiteralPath $entry.Value -PathType Leaf)) {
throw "M48N source $($entry.Key) is missing"
}
}
if ((Get-Sha256 $source.Video) -cne "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8") {
throw "RAVNOVES00 video SHA-256 changed"
}
if ((Get-Sha256 $source.Mask) -cne "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63") {
throw "valid-FOV mask SHA-256 changed"
}
$media = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1"
) "PyAV dependency" $false
$opencv = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages"
) "OpenCV dependency" $false
$pillow = Resolve-DDirectory (
"D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1"
) "Pillow dependency" $false
$image = (
"nvcr.io/nvidia/tritonserver:26.06-py3@" +
"sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
)
& docker image inspect $image *> $null
Assert-LastExitCode "pinned M48N image inspection"
$canonicalTriton = Get-Container "ndc-mission-core-triton"
if (-not $canonicalTriton.State.Running -or $canonicalTriton.State.Health.Status -cne "healthy") {
throw "Canonical Triton must remain healthy during M48N shadow"
}
$canonicalTritonId = [string]$canonicalTriton.Id
$tritonName = "ndc-mission-core-m48n-native-reference-graph-triton"
$graphName = "ndc-mission-core-m48n-native-reference-graph"
foreach ($name in @($tritonName, $graphName)) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
throw "M48N candidate container $name already exists"
}
}
try {
& docker create `
--name $tritonName `
--label "com.nodedc.product=mission-core" `
--label "com.nodedc.stack=ndc-mission-core-compute" `
--label "com.nodedc.role=bounded-native-rf-detr-reference-graph-triton" `
--label "com.nodedc.managed-by=codex-bounded-experiment" `
--read-only `
--security-opt "no-new-privileges:true" `
--cap-drop ALL `
--pids-limit 512 `
--shm-size 1g `
--gpus all `
--tmpfs "/tmp:rw,noexec,nosuid,size=2g" `
--health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" `
--health-interval 5s `
--health-timeout 3s `
--health-start-period 20s `
--health-retries 24 `
-v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") `
$image `
tritonserver `
--model-repository=/models `
--model-control-mode=explicit `
--load-model=rf_detr_large_native_kb4 `
--disable-auto-complete-config `
--strict-readiness=true `
--exit-on-error=true `
--allow-http=true `
--allow-grpc=false `
--allow-metrics=false *> $null
Assert-LastExitCode "M48N Triton creation"
& docker start $tritonName *> $null
Assert-LastExitCode "M48N Triton start"
$ready = $false
foreach ($attempt in 1..60) {
Start-Sleep -Seconds 2
$candidateContainer = Get-Container $tritonName
if (-not $candidateContainer.State.Running) {
& docker logs $tritonName
throw "M48N Triton stopped during startup"
}
if ($candidateContainer.State.Health.Status -ceq "healthy") {
$ready = $true
break
}
}
if (-not $ready) { throw "M48N Triton did not become healthy" }
if (@((Get-Container $tritonName).HostConfig.PortBindings.PSObject.Properties).Count -ne 0) {
throw "M48N Triton published a host port"
}
$arguments = @(
"run", "--name", $graphName,
"--label", "com.nodedc.product=mission-core",
"--label", "com.nodedc.stack=ndc-mission-core-compute",
"--label", "com.nodedc.role=bounded-native-rf-detr-reference-graph",
"--label", "com.nodedc.managed-by=codex-bounded-experiment",
"--network", ("container:{0}" -f $tritonName),
"--read-only",
"--security-opt", "no-new-privileges:true",
"--cap-drop", "ALL",
"--pids-limit", "256",
"--gpus", "all",
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
"-e", "PYTHONDONTWRITEBYTECODE=1",
"-e", (
"PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:" +
"/opt/media:/opt/opencv:/opt/pillow"
),
"-v", ((Convert-ToDockerPath $release) + ":/release:ro"),
"-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"),
"-v", ((Convert-ToDockerPath $media) + ":/opt/media:ro"),
"-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"),
"-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"),
"-v", ((Convert-ToDockerPath $source.CameraIndex) + ":/source/camera-index.jsonl:ro"),
"-v", ((Convert-ToDockerPath $source.SourcePack) + ":/source/source-pack.npz:ro"),
"-v", ((Convert-ToDockerPath $source.LocalSurface) + ":/source/local-surface.npz:ro"),
"-v", ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro"),
"-v", ((Convert-ToDockerPath $source.Mask) + ":/source/mask.png:ro"),
"--entrypoint", "python3",
$image,
"/release/run_m48s_reference_graph_shadow_worker.py",
"--graph-config", "/release/m48n-rf-detr-native-reference-graph-shadow-v0.json",
"--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json",
"--detector-profile", "/release/rf-detr-large-native-kb4-risk-shadow-v0.json",
"--geometry-profile", "/release/m4-geometry-association-v1.json",
"--temporal-motion-profile", "/release/m4-temporal-motion-v1.json",
"--rolling-map-profile", "/release/m4-rolling-local-map-v1.json",
"--threat-profile", "/release/m4-replay-threat-v3.json",
"--camera-index", "/source/camera-index.jsonl",
"--source-pack", "/source/source-pack.npz",
"--local-surface", "/source/local-surface.npz",
"--video", "/source/right.mp4",
"--valid-fov-mask", "/source/mask.png",
"--triton-origin", "http://127.0.0.1:8000",
"--loops", "1",
"--maximum-frames", ([string]$MaximumFrames),
"--source-rate-hz", ([string]::Format(
[Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz
)),
"--minimum-delivery-ratio", ([string]::Format(
[Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumDeliveryRatio
)),
"--minimum-effective-world-state-fps", ([string]::Format(
[Globalization.CultureInfo]::InvariantCulture,
"{0:R}",
$MinimumEffectiveWorldStateFps
)),
"--maximum-world-state-completion-p95-ms", ([string]::Format(
[Globalization.CultureInfo]::InvariantCulture,
"{0:R}",
$MaximumWorldStateCompletionP95Ms
)),
"--load-purpose", "production-rate",
"--runtime-artifact-sha256", $ExpectedWheelSha256,
"--runner-sha256", $runnerSha256,
"--output", "/output/result.json",
"--progress", "/output/progress.jsonl",
"--frame-ledger", "/output/frames.jsonl"
)
& docker @arguments
Assert-LastExitCode "M48N native complete reference graph shadow"
foreach ($name in @("result.json", "frames.jsonl", "progress.jsonl")) {
if (-not (Test-Path -LiteralPath (Join-Path $runOutput $name) -PathType Leaf)) {
throw "M48N graph artifact $name was not written"
}
}
} finally {
foreach ($name in @($graphName, $tritonName)) {
if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") {
& docker rm -f $name *> $null
}
}
$canonicalAfter = Get-Container "ndc-mission-core-triton"
if (
$canonicalAfter.Id -cne $canonicalTritonId -or
-not $canonicalAfter.State.Running -or
$canonicalAfter.State.Health.Status -cne "healthy"
) {
throw "Canonical Triton changed during M48N shadow"
}
}
Write-Output ("M48N_NATIVE_REFERENCE_GRAPH_RESULT={0}" -f (Join-Path $runOutput "result.json"))
Write-Output "CANONICAL_TRITON_ACTION=none"
Write-Output "PRODUCTION_ACCEPTED=false"
src/k1link/perception/detector.py
+181
View File
@@ -14,6 +14,15 @@ from numpy.typing import NDArray
from .contracts import BoundingRegion2D, ObjectProposal2D
from .providers import SourcePacket
from .rf_detr_native_object_detector import (
RF_DETR_NATIVE_CONFIG,
RF_DETR_NATIVE_MODEL_ID,
RF_DETR_NATIVE_MODEL_VERSION,
NativeRfDetrConfig,
NativeRfDetrInferenceBackend,
postprocess_native_rf_detr,
prepare_raw_kb4_rf_detr_native,
)
from .rf_detr_object_detector import (
RF_DETR_CONFIG,
RF_DETR_MODEL_ID,
@@ -45,6 +54,15 @@ FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1"
RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0"
RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}"
RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0"
RF_DETR_NATIVE_SHADOW_PROVIDER_ID: Final = (
"triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0"
)
RF_DETR_NATIVE_SHADOW_MODEL_ID: Final = (
f"{RF_DETR_NATIVE_MODEL_ID}:{RF_DETR_NATIVE_MODEL_VERSION}"
)
RF_DETR_NATIVE_SHADOW_PREPROCESS_ID: Final = (
"raw-kb4-uint8-fused-mask-rgb-pad8-imagenet-trt/v0"
)
class DetectorProviderError(RuntimeError):
@@ -414,6 +432,164 @@ def proposals_from_rf_detr_detections(
)
class NativeRfDetrShadowDetectorProvider:
"""Emit risk classes from one exact-raster native RF-DETR inference pass."""
provider_id: str = RF_DETR_NATIVE_SHADOW_PROVIDER_ID
def __init__(
self,
*,
mask: NDArray[np.bool_],
backend: NativeRfDetrInferenceBackend,
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
clock_ns: Callable[[], int] = time.perf_counter_ns,
timing_observer: DetectorTimingObserver | None = None,
) -> None:
if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask):
raise DetectorProviderError("native RF-DETR valid-FOV mask is incompatible")
self.mask = np.asarray(mask, dtype=np.bool_)
self.backend = backend
self.config = config
self._clock_ns = clock_ns
self.timing_observer = timing_observer
self._lock = Lock()
self._input_frames = 0
self._completed_frames = 0
self._failed_frames = 0
self._zero_proposal_frames = 0
self._proposal_count = 0
self._rejected: Counter[str] = Counter()
self._core_duration_ns = 0
self._warmup_started = False
self._warmup_snapshot: DetectorWarmupSnapshot | None = None
def warm_up(self) -> DetectorWarmupSnapshot:
"""Prime raw transport and postprocessing before source admission."""
with self._lock:
if self._warmup_snapshot is not None:
return self._warmup_snapshot
if self._warmup_started:
raise DetectorProviderError("native RF-DETR warmup is already in progress")
self._warmup_started = True
started_ns = int(self._clock_ns())
try:
image = np.zeros(
(self.config.source_height, self.config.source_width, 3),
dtype=np.uint8,
)
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
preprocessed_ns = int(self._clock_ns())
output = self.backend.infer(tensor)
inferred_ns = int(self._clock_ns())
postprocess_native_rf_detr(output, self.mask, config=self.config)
completed_ns = int(self._clock_ns())
except Exception:
with self._lock:
self._warmup_started = False
raise
snapshot = DetectorWarmupSnapshot(
completed=True,
inference_passes=1,
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
total_duration_ns=max(0, completed_ns - started_ns),
)
with self._lock:
self._warmup_snapshot = snapshot
return snapshot
def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]:
payload = packet.image_payload
with self._lock:
self._input_frames += 1
started_ns = int(self._clock_ns())
try:
if not isinstance(payload, np.ndarray):
raise DetectorProviderError(
"native RF-DETR requires a decoded BGR image payload"
)
image = np.asarray(payload)
if image.dtype != np.uint8:
raise DetectorProviderError("decoded BGR image must be uint8")
tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config)
preprocessed_ns = (
int(self._clock_ns()) if self.timing_observer is not None else started_ns
)
output = self.backend.infer(tensor)
inferred_ns = (
int(self._clock_ns()) if self.timing_observer is not None else preprocessed_ns
)
postprocessed = postprocess_native_rf_detr(
output,
self.mask,
config=self.config,
)
proposals = proposals_from_native_rf_detr_detections(
packet,
postprocessed.detections,
)
except Exception:
with self._lock:
self._failed_frames += 1
self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns)
raise
completed_ns = int(self._clock_ns())
with self._lock:
self._completed_frames += 1
self._proposal_count += len(proposals)
self._zero_proposal_frames += not proposals
self._rejected.update(dict(postprocessed.rejected))
self._core_duration_ns += max(0, completed_ns - started_ns)
if self.timing_observer is not None:
self.timing_observer(
DetectorFrameTiming(
sequence=packet.envelope.sequence,
preprocess_duration_ns=max(0, preprocessed_ns - started_ns),
inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns),
postprocess_duration_ns=max(0, completed_ns - inferred_ns),
total_duration_ns=max(0, completed_ns - started_ns),
)
)
return proposals
def snapshot(self) -> DetectorProviderSnapshot:
with self._lock:
return DetectorProviderSnapshot(
input_frames=self._input_frames,
completed_frames=self._completed_frames,
failed_frames=self._failed_frames,
zero_proposal_frames=self._zero_proposal_frames,
proposal_count=self._proposal_count,
rejected=tuple(sorted(self._rejected.items())),
core_duration_ns=self._core_duration_ns,
)
def proposals_from_native_rf_detr_detections(
packet: SourcePacket,
detections: tuple[RfDetrDetection, ...],
) -> tuple[ObjectProposal2D, ...]:
envelope = packet.envelope
return tuple(
ObjectProposal2D(
proposal_id=f"proposal-{envelope.sequence}-{index}",
source_id=envelope.source_id,
frame_id=envelope.frame_id,
region=BoundingRegion2D(*detection.bbox_xyxy),
objectness=detection.score,
provider_id=RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
model_id=RF_DETR_NATIVE_SHADOW_MODEL_ID,
preprocess_id=RF_DETR_NATIVE_SHADOW_PREPROCESS_ID,
semantic_hint=detection.label,
provider_tracklet=None,
)
for index, detection in enumerate(detections)
)
__all__ = [
"ALL_COCO_YOLOX_PROVIDER_ID",
"FROZEN_YOLOX_MODEL_ID",
@@ -422,6 +598,9 @@ __all__ = [
"RF_DETR_SHADOW_MODEL_ID",
"RF_DETR_SHADOW_PREPROCESS_ID",
"RF_DETR_SHADOW_PROVIDER_ID",
"RF_DETR_NATIVE_SHADOW_MODEL_ID",
"RF_DETR_NATIVE_SHADOW_PREPROCESS_ID",
"RF_DETR_NATIVE_SHADOW_PROVIDER_ID",
"DetectorProviderError",
"DetectorProviderSnapshot",
"DetectorFrameTiming",
@@ -429,7 +608,9 @@ __all__ = [
"DetectorWarmupSnapshot",
"AllCocoYoloxDetectorProvider",
"FrozenYoloxDetectorProvider",
"NativeRfDetrShadowDetectorProvider",
"RfDetrShadowDetectorProvider",
"proposals_from_detections",
"proposals_from_native_rf_detr_detections",
"proposals_from_rf_detr_detections",
]
src/k1link/perception/m48s_reference_graph_runtime.py
+67 -20
View File
@@ -8,12 +8,15 @@ from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from threading import Event
from typing import Literal
from .baseline import load_m4_baseline
from .detector import (
RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
RF_DETR_SHADOW_PROVIDER_ID,
DetectorTimingObserver,
DetectorWarmupSnapshot,
NativeRfDetrShadowDetectorProvider,
RfDetrShadowDetectorProvider,
)
from .geometry import (
@@ -41,6 +44,12 @@ from .recorded_source import (
SourcePacingObserver,
)
from .reference_graph_runtime import ReferenceGraphRuntimePaths
from .rf_detr_native_object_detector import (
RF_DETR_NATIVE_ENGINE_SHA256,
RF_DETR_NATIVE_MODEL_ID,
RF_DETR_NATIVE_MODEL_VERSION,
TritonNativeRfDetrHttpInferenceBackend,
)
from .rf_detr_object_detector import (
RF_DETR_ENGINE_SHA256,
RF_DETR_MODEL_ID,
@@ -66,13 +75,18 @@ class M48sReferenceGraphRuntime:
"""Own one RF-DETR shadow graph and its persistent inference transport."""
graph: ReferencePerceptionGraphV2
inference_backend: TritonRfDetrHttpInferenceBackend
inference_backend: (
TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend
)
source_prefetch: PrefetchedRecordedImageDecoder
_preparation_stop_event: Event = field(default_factory=Event)
def warm_up_detector(self) -> DetectorWarmupSnapshot:
detector = self.graph.detector
if not isinstance(detector, RfDetrShadowDetectorProvider):
if not isinstance(
detector,
(RfDetrShadowDetectorProvider, NativeRfDetrShadowDetectorProvider),
):
raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup")
return detector.warm_up()
@@ -125,7 +139,7 @@ def build_m48s_reference_graph_runtime(
ProviderRole.THREAT: paths.threat_profile,
}
_validate_provider_digests(config, pinned_files)
_validate_detector_profile(detector_profile)
detector_variant = _validate_detector_profile(detector_profile)
load_m4_baseline(paths.baseline_profile)
geometry_profile = load_geometry_profile(paths.geometry_profile)
@@ -161,7 +175,22 @@ def build_m48s_reference_graph_runtime(
)
if maximum_frames is not None:
source = _LimitedSource(source, maximum_frames)
backend = TritonRfDetrHttpInferenceBackend(triton_origin)
if detector_variant == "legacy-704":
backend: (
TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend
) = TritonRfDetrHttpInferenceBackend(triton_origin)
detector = RfDetrShadowDetectorProvider(
mask=load_valid_fov_mask(paths.valid_fov_mask),
backend=backend,
timing_observer=detector_timing_observer,
)
else:
backend = TritonNativeRfDetrHttpInferenceBackend(triton_origin)
detector = NativeRfDetrShadowDetectorProvider(
mask=load_valid_fov_mask(paths.valid_fov_mask),
backend=backend,
timing_observer=detector_timing_observer,
)
try:
store = RecordedGeometryStore(
source_pack_path=paths.source_pack,
@@ -175,11 +204,7 @@ def build_m48s_reference_graph_runtime(
graph = ReferencePerceptionGraphV2(
config=config,
source=source,
detector=RfDetrShadowDetectorProvider(
mask=load_valid_fov_mask(paths.valid_fov_mask),
backend=backend,
timing_observer=detector_timing_observer,
),
detector=detector,
geometry=Ravnoves00GeometryAssociationProvider(store=store),
temporal=BoundedSpatialTemporalProvider(
point_resolver=store,
@@ -249,7 +274,7 @@ def _validate_provider_digests(
raise M48sReferenceGraphRuntimeError(f"{role.value} provider profile digest changed")
def _validate_detector_profile(path: Path) -> None:
def _validate_detector_profile(path: Path) -> Literal["legacy-704", "native-kb4"]:
try:
document = json.loads(path.resolve(strict=True).read_text("utf-8"))
model = document["model"]
@@ -257,15 +282,7 @@ def _validate_detector_profile(path: Path) -> None:
authority = document["authority"]
except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc:
raise M48sReferenceGraphRuntimeError("RF-DETR profile is incomplete") from exc
if (
document.get("schema_version") != "missioncore.rf-detr-risk-shadow-profile/v0"
or document.get("provider_id") != RF_DETR_SHADOW_PROVIDER_ID
or model.get("model_id") != RF_DETR_MODEL_ID
or model.get("model_version") != RF_DETR_MODEL_VERSION
or model.get("worker_006_rtx4090_tensorrt_11_engine_sha256") != RF_DETR_ENGINE_SHA256
or status.get("detector_load_gate_passed") is not True
or status.get("production_accepted") is not False
or any(
authority_false = not any(
authority.get(key) is not False
for key in (
"candidate_accepted",
@@ -274,7 +291,37 @@ def _validate_detector_profile(path: Path) -> None:
"navigation_or_safety_accepted",
)
)
):
legacy = (
document.get("schema_version") == "missioncore.rf-detr-risk-shadow-profile/v0"
and document.get("provider_id") == RF_DETR_SHADOW_PROVIDER_ID
and model.get("model_id") == RF_DETR_MODEL_ID
and model.get("model_version") == RF_DETR_MODEL_VERSION
and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256")
== RF_DETR_ENGINE_SHA256
and status.get("detector_load_gate_passed") is True
and status.get("production_accepted") is False
and authority_false
)
native = (
document.get("schema_version")
== "missioncore.rf-detr-native-risk-shadow-profile/v0"
and document.get("provider_id") == RF_DETR_NATIVE_SHADOW_PROVIDER_ID
and model.get("model_id") == RF_DETR_NATIVE_MODEL_ID
and model.get("model_version") == RF_DETR_NATIVE_MODEL_VERSION
and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256")
== RF_DETR_NATIVE_ENGINE_SHA256
and status.get("native_tensor_parity_passed") is True
and status.get("full_ravnoves00_runtime_gate_passed") is True
and status.get("legacy_704_box_agreement_gate_passed") is False
and status.get("integrated_world_state_gate_passed") is False
and status.get("production_accepted") is False
and authority_false
)
if legacy:
return "legacy-704"
if native:
return "native-kb4"
else:
raise M48sReferenceGraphRuntimeError("RF-DETR shadow profile identity changed")
src/k1link/perception/rf_detr_native_object_detector.py
+364
View File
@@ -0,0 +1,364 @@
"""Native raw-KB4 RF-DETR-L TensorRT transport and postprocessing.
The TensorRT engine owns valid-FOV masking, BGR-to-RGB conversion, eight
bottom padding rows and ImageNet normalization. The client sends the exact
800x600 UINT8 KB4 raster and performs no geometric resampling.
"""
from __future__ import annotations
import http.client
import json
import math
import urllib.parse
from collections import Counter
from dataclasses import dataclass
from typing import Final, Protocol, cast
import numpy as np
from numpy.typing import NDArray
from .rf_detr_object_detector import (
COCO_SPARSE_TO_CONTIGUOUS,
RISK_CLASS_IDS,
RfDetrDetection,
RfDetrPostprocessResult,
RfDetrRawOutput,
)
from .yolox_object_detector import COCO_CLASSES, YOLOX_VALID_FOV_SHA256
RF_DETR_NATIVE_MODEL_ID: Final = "rf_detr_large_native_kb4"
RF_DETR_NATIVE_MODEL_VERSION: Final = 1
RF_DETR_NATIVE_CHECKPOINT_SHA256: Final = (
"0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38"
)
RF_DETR_NATIVE_CORE_ONNX_SHA256: Final = (
"62e549748a1d17646b90ad06d3ac8a1b79595b7e9270cac4564418f023079176"
)
RF_DETR_NATIVE_FP16_ONNX_SHA256: Final = (
"00b29fa2ff3d5fca730ebf3b8c33b10e9d690cc97bbbbfa5563d6e0abf210999"
)
RF_DETR_NATIVE_WRAPPED_ONNX_SHA256: Final = (
"acdd01623a00d100331473c0a99eab1e5adf33117cbab900c8ae078edd4aa346"
)
RF_DETR_NATIVE_ENGINE_SHA256: Final = (
"b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695"
)
RF_DETR_NATIVE_VALID_FOV_SHA256: Final = YOLOX_VALID_FOV_SHA256
class NativeRfDetrDetectorError(RuntimeError):
"""The native RF-DETR profile, tensor or response is incompatible."""
@dataclass(frozen=True, slots=True)
class NativeRfDetrConfig:
source_width: int = 800
source_height: int = 600
model_width: int = 800
model_height: int = 608
bottom_padding_rows: int = 8
fill_value: int = 114
minimum_score: float = 0.25
target_class_ids: tuple[int, ...] = RISK_CLASS_IDS
maximum_detections: int = 300
minimum_box_area_pixels: float = 64.0
maximum_box_area_fraction: float = 0.5
minimum_valid_fov_fraction: float = 0.5
require_center_inside_valid_fov: bool = True
def __post_init__(self) -> None:
if (
self.source_width,
self.source_height,
self.model_width,
self.model_height,
self.bottom_padding_rows,
self.fill_value,
self.minimum_score,
self.target_class_ids,
self.maximum_detections,
self.minimum_box_area_pixels,
self.maximum_box_area_fraction,
self.minimum_valid_fov_fraction,
self.require_center_inside_valid_fov,
) != (
800,
600,
800,
608,
8,
114,
0.25,
RISK_CLASS_IDS,
300,
64.0,
0.5,
0.5,
True,
):
raise NativeRfDetrDetectorError(
"native RF-DETR shadow profile cannot be tuned in place"
)
RF_DETR_NATIVE_CONFIG: Final = NativeRfDetrConfig()
class NativeRfDetrInferenceBackend(Protocol):
def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput: ...
class TritonNativeRfDetrHttpInferenceBackend:
"""Persistent Triton V2 HTTP transport for exact raw UINT8 KB4 frames."""
def __init__(self, endpoint: str, *, timeout_seconds: float = 60.0) -> None:
parsed = urllib.parse.urlsplit(endpoint)
if (
parsed.scheme != "http"
or not parsed.hostname
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
):
raise NativeRfDetrDetectorError(
"Triton endpoint must be an explicit HTTP origin"
)
if not math.isfinite(timeout_seconds) or timeout_seconds <= 0:
raise NativeRfDetrDetectorError("Triton timeout must be positive")
self.path = (
f"{parsed.path.rstrip('/')}/v2/models/{RF_DETR_NATIVE_MODEL_ID}"
f"/versions/{RF_DETR_NATIVE_MODEL_VERSION}/infer"
)
self.connection = http.client.HTTPConnection(
parsed.hostname,
parsed.port or 80,
timeout=timeout_seconds,
)
def close(self) -> None:
self.connection.close()
def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput:
contiguous = np.ascontiguousarray(tensor, dtype=np.uint8)
if contiguous.shape != (1, 600, 800, 3):
raise NativeRfDetrDetectorError(
"Triton native RF-DETR input tensor is incompatible"
)
binary = contiguous.tobytes()
header = {
"inputs": [
{
"name": "raw_kb4_bgr",
"shape": [1, 600, 800, 3],
"datatype": "UINT8",
"parameters": {"binary_data_size": len(binary)},
}
],
"outputs": [
{"name": "dets", "parameters": {"binary_data": True}},
{"name": "labels", "parameters": {"binary_data": True}},
],
}
encoded = json.dumps(header, sort_keys=True, separators=(",", ":")).encode()
self.connection.request(
"POST",
self.path,
body=encoded + binary,
headers={
"Content-Type": "application/octet-stream",
"Inference-Header-Content-Length": str(len(encoded)),
},
)
response = self.connection.getresponse()
payload = response.read()
if response.status != 200:
raise NativeRfDetrDetectorError(
f"Triton native RF-DETR inference failed with HTTP {response.status}"
)
header_value = response.getheader("Inference-Header-Content-Length")
try:
header_length = int(header_value or "")
descriptor = json.loads(payload[:header_length])
outputs = descriptor["outputs"]
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
raise NativeRfDetrDetectorError(
"Triton native RF-DETR output descriptor is invalid"
) from exc
if not isinstance(outputs, list) or len(outputs) != 2:
raise NativeRfDetrDetectorError(
"Triton native RF-DETR output count changed"
)
offset = header_length
arrays: dict[str, NDArray[np.float16]] = {}
for output, expected_name, expected_shape in zip(
outputs,
("dets", "labels"),
((1, 300, 4), (1, 300, 91)),
strict=True,
):
try:
name = output["name"]
datatype = output["datatype"]
shape = tuple(int(value) for value in output["shape"])
byte_length = int(output["parameters"]["binary_data_size"])
except (KeyError, TypeError, ValueError) as exc:
raise NativeRfDetrDetectorError(
"Triton native RF-DETR output descriptor is incomplete"
) from exc
expected_bytes = math.prod(expected_shape) * np.dtype("<f2").itemsize
if (
name != expected_name
or datatype != "FP16"
or shape != expected_shape
or byte_length != expected_bytes
or offset + byte_length > len(payload)
):
raise NativeRfDetrDetectorError(
"Triton native RF-DETR output identity changed"
)
array = np.frombuffer(payload[offset : offset + byte_length], dtype="<f2")
arrays[name] = np.asarray(array.reshape(shape), dtype=np.float16)
offset += byte_length
if offset != len(payload):
raise NativeRfDetrDetectorError(
"Triton native RF-DETR output byte length changed"
)
return RfDetrRawOutput(boxes=arrays["dets"], logits=arrays["labels"])
def prepare_raw_kb4_rf_detr_native(
image_bgr: NDArray[np.uint8],
*,
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
) -> NDArray[np.uint8]:
"""Expose the exact raw KB4 raster as UINT8 NHWC without image transforms."""
if image_bgr.shape != (config.source_height, config.source_width, 3):
raise NativeRfDetrDetectorError("raw KB4 image raster changed")
if image_bgr.dtype != np.uint8:
raise NativeRfDetrDetectorError("raw KB4 image must be uint8")
return np.ascontiguousarray(image_bgr[None], dtype=np.uint8)
def postprocess_native_rf_detr(
output: RfDetrRawOutput,
mask: NDArray[np.bool_],
*,
config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG,
) -> RfDetrPostprocessResult:
if output.boxes.shape != (1, 300, 4) or output.logits.shape != (1, 300, 91):
raise NativeRfDetrDetectorError("native RF-DETR output shapes are incompatible")
if output.boxes.dtype != np.float16 or output.logits.dtype != np.float16:
raise NativeRfDetrDetectorError("native RF-DETR output types are incompatible")
if not np.isfinite(output.boxes).all() or not np.isfinite(output.logits).all():
raise NativeRfDetrDetectorError("native RF-DETR output contains non-finite values")
if mask.shape != (config.source_height, config.source_width) or mask.dtype != np.bool_:
raise NativeRfDetrDetectorError("valid-FOV mask is incompatible")
logits = output.logits[0].astype(np.float32)
probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits, -80.0, 80.0)))
flattened = probabilities.reshape(-1)
topk = np.argsort(-flattened, kind="stable")[: config.maximum_detections]
integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1)
rejected: Counter[str] = Counter()
result: list[RfDetrDetection] = []
for flat_index in topk:
score = float(flattened[flat_index])
if score <= config.minimum_score:
continue
query_index = int(flat_index // output.logits.shape[2])
sparse_class_id = int(flat_index % output.logits.shape[2])
class_id = COCO_SPARSE_TO_CONTIGUOUS.get(sparse_class_id)
if class_id is None:
rejected["unmapped-class-slot"] += 1
continue
if class_id not in config.target_class_ids:
rejected["non-risk-class"] += 1
continue
center_x, center_y, box_width, box_height = (
float(value) for value in output.boxes[0, query_index].astype(np.float32)
)
box = np.asarray(
(
(center_x - box_width / 2.0) * config.model_width,
(center_y - box_height / 2.0) * config.model_height,
(center_x + box_width / 2.0) * config.model_width,
(center_y + box_height / 2.0) * config.model_height,
),
dtype=np.float32,
)
box[[0, 2]] = np.clip(box[[0, 2]], 0, config.source_width)
box[[1, 3]] = np.clip(box[[1, 3]], 0, config.source_height)
fraction, center_inside, area = _valid_fraction(box, integral)
if area < config.minimum_box_area_pixels:
rejected["small-box"] += 1
continue
if area / (config.source_width * config.source_height) > (
config.maximum_box_area_fraction
):
rejected["large-box"] += 1
continue
if fraction < config.minimum_valid_fov_fraction:
rejected["outside-valid-fov"] += 1
continue
if config.require_center_inside_valid_fov and not center_inside:
rejected["center-outside-valid-fov"] += 1
continue
result.append(
RfDetrDetection(
class_id=class_id,
label=COCO_CLASSES[class_id],
score=round(score, 9),
bbox_xyxy=cast(
tuple[float, float, float, float],
tuple(round(float(value), 6) for value in box),
),
valid_fov_fraction=round(fraction, 6),
)
)
result.sort(key=lambda item: (-item.score, item.class_id))
return RfDetrPostprocessResult(tuple(result), tuple(sorted(rejected.items())))
def _valid_fraction(
box: NDArray[np.float32], integral: NDArray[np.int64]
) -> tuple[float, bool, float]:
height = integral.shape[0] - 1
width = integral.shape[1] - 1
x1 = int(np.clip(math.floor(float(box[0])), 0, width))
y1 = int(np.clip(math.floor(float(box[1])), 0, height))
x2 = int(np.clip(math.ceil(float(box[2])), 0, width))
y2 = int(np.clip(math.ceil(float(box[3])), 0, height))
area = float(max(0, x2 - x1) * max(0, y2 - y1))
if area <= 0:
return 0.0, False, 0.0
inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1]
center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1))
center_y = int(np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1))
center_inside = bool(
integral[center_y + 1, center_x + 1]
- integral[center_y, center_x + 1]
- integral[center_y + 1, center_x]
+ integral[center_y, center_x]
)
return float(inside) / area, center_inside, area
__all__ = [
"RF_DETR_NATIVE_CHECKPOINT_SHA256",
"RF_DETR_NATIVE_CONFIG",
"RF_DETR_NATIVE_CORE_ONNX_SHA256",
"RF_DETR_NATIVE_ENGINE_SHA256",
"RF_DETR_NATIVE_FP16_ONNX_SHA256",
"RF_DETR_NATIVE_MODEL_ID",
"RF_DETR_NATIVE_MODEL_VERSION",
"RF_DETR_NATIVE_VALID_FOV_SHA256",
"RF_DETR_NATIVE_WRAPPED_ONNX_SHA256",
"NativeRfDetrConfig",
"NativeRfDetrDetectorError",
"NativeRfDetrInferenceBackend",
"TritonNativeRfDetrHttpInferenceBackend",
"postprocess_native_rf_detr",
"prepare_raw_kb4_rf_detr_native",
]
tests/test_m48s_reference_graph_shadow.py
+53 -1
View File
@@ -4,7 +4,10 @@ import hashlib
import json
from pathlib import Path
from k1link.perception.detector import RF_DETR_SHADOW_PROVIDER_ID
from k1link.perception.detector import (
RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
RF_DETR_SHADOW_PROVIDER_ID,
)
from k1link.perception.m48s_advisory import (
AdvisoryFamily,
advisory_policy_matrix,
@@ -17,6 +20,10 @@ GRAPH_CONFIG = (
REPOSITORY_ROOT
/ "config/perception/m48s-rf-detr-reference-graph-shadow-v0.json"
)
NATIVE_GRAPH_CONFIG = (
REPOSITORY_ROOT
/ "config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json"
)
def test_m48s_reference_graph_replaces_only_the_detector_pin() -> None:
@@ -64,6 +71,51 @@ def test_m48s_reference_graph_pins_every_profile_digest() -> None:
assert pins[role].sha256 == hashlib.sha256(payload).hexdigest()
def test_m48n_native_reference_graph_replaces_only_the_detector_pin() -> None:
native = ReferencePerceptionGraphConfigV2.from_dict(
json.loads(NATIVE_GRAPH_CONFIG.read_text("utf-8"))
)
legacy = ReferencePerceptionGraphConfigV2.from_dict(
json.loads(GRAPH_CONFIG.read_text("utf-8"))
)
native_pins = {item.role: item for item in native.providers}
legacy_pins = {item.role: item for item in legacy.providers}
assert native.graph_id == legacy.graph_id == "reference-perception-graph/v2"
assert native.source_profile_id == legacy.source_profile_id
assert native.queues == legacy.queues
assert native.authority == legacy.authority
assert (
native_pins[ProviderRole.DETECTOR].provider_id
== RF_DETR_NATIVE_SHADOW_PROVIDER_ID
)
assert all(
native_pins[role] == legacy_pins[role]
for role in ProviderRole
if role is not ProviderRole.DETECTOR
)
def test_m48n_native_reference_graph_pins_every_profile_digest() -> None:
config = ReferencePerceptionGraphConfigV2.from_dict(
json.loads(NATIVE_GRAPH_CONFIG.read_text("utf-8"))
)
paths = {
ProviderRole.SOURCE: "m4-recorded-realtime-baseline-v1.json",
ProviderRole.DETECTOR: "rf-detr-large-native-kb4-risk-shadow-v0.json",
ProviderRole.GEOMETRY: "m4-geometry-association-v1.json",
ProviderRole.TEMPORAL: "m4-temporal-motion-v1.json",
ProviderRole.MOTION: "m4-temporal-motion-v1.json",
ProviderRole.ROLLING: "m4-rolling-local-map-v1.json",
ProviderRole.THREAT: "m4-replay-threat-v3.json",
}
pins = {item.role: item for item in config.providers}
for role, name in paths.items():
payload = (REPOSITORY_ROOT / "config/perception" / name).read_bytes()
assert pins[role].sha256 == hashlib.sha256(payload).hexdigest()
def test_m48s_advisory_policy_is_bounded_distinct_and_commandless() -> None:
matrix = advisory_policy_matrix()
tests/test_rf_detr_native_detector_provider.py
+236
View File
@@ -0,0 +1,236 @@
from __future__ import annotations
import json
import math
from pathlib import Path
import numpy as np
import pytest
from numpy.typing import NDArray
from k1link.perception.contracts import (
ClockBasis,
ModalityOutcome,
ModalityStatus,
SourceEnvelope,
TimestampBundle,
)
from k1link.perception.detector import (
RF_DETR_NATIVE_SHADOW_MODEL_ID,
RF_DETR_NATIVE_SHADOW_PREPROCESS_ID,
RF_DETR_NATIVE_SHADOW_PROVIDER_ID,
DetectorFrameTiming,
NativeRfDetrShadowDetectorProvider,
)
from k1link.perception.m48s_reference_graph_runtime import _validate_detector_profile
from k1link.perception.providers import SourcePacket
from k1link.perception.rf_detr_native_object_detector import (
RF_DETR_NATIVE_CONFIG,
RF_DETR_NATIVE_ENGINE_SHA256,
NativeRfDetrConfig,
NativeRfDetrDetectorError,
TritonNativeRfDetrHttpInferenceBackend,
postprocess_native_rf_detr,
prepare_raw_kb4_rf_detr_native,
)
from k1link.perception.rf_detr_object_detector import RfDetrRawOutput
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
def _status() -> ModalityStatus:
return ModalityStatus(True, ModalityOutcome.AVAILABLE, "test-available")
def _packet(sequence: int, image: object) -> SourcePacket:
return SourcePacket(
envelope=SourceEnvelope(
source_id="RAVNOVES00",
session_id="20260720T065719Z_viewer_live",
frame_id=f"frame-{sequence:06d}",
sequence=sequence,
timestamps=TimestampBundle(
utc_ns=1_000 + sequence,
monotonic_ns=2_000 + sequence,
source_ns=3_000 + sequence,
clock_basis=ClockBasis.RECORDED_HOST,
),
source_age_ns=0,
binding_reason="test-recorded-source",
calibration_id="camera-1-kb4-test",
representation_id="registered-map-increment-v1",
image=_status(),
registered_point_increment=_status(),
pose=_status(),
),
image_payload=image,
registered_point_increment_payload=("points", sequence),
pose_payload=("pose", sequence),
)
def _output() -> RfDetrRawOutput:
boxes = np.zeros((1, 300, 4), dtype=np.float16)
logits = np.full((1, 300, 91), -20.0, dtype=np.float16)
boxes[0, 0] = (0.5, 0.5, 0.25, 0.25)
logits[0, 0, 18] = np.float16(math.log(3.0)) # dog, score 0.75
boxes[0, 1] = (0.25, 0.25, 0.1, 0.2)
logits[0, 1, 1] = np.float16(math.log(4.0)) # person, score 0.80
boxes[0, 2] = (0.75, 0.25, 0.1, 0.2)
logits[0, 2, 62] = np.float16(math.log(9.0)) # chair, non-risk
return RfDetrRawOutput(boxes=boxes, logits=logits)
class _Backend:
def __init__(self, output: RfDetrRawOutput) -> None:
self.output = output
self.calls = 0
def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput:
assert tensor.shape == (1, 600, 800, 3)
assert tensor.dtype == np.uint8
assert tensor.flags.c_contiguous
self.calls += 1
return self.output
def test_native_prepare_preserves_every_raw_pixel_without_geometric_transform() -> None:
raster = np.arange(600 * 800 * 3, dtype=np.uint8).reshape(600, 800, 3)
non_contiguous = raster[:, ::-1]
tensor = prepare_raw_kb4_rf_detr_native(non_contiguous)
assert tensor.shape == (1, 600, 800, 3)
assert tensor.dtype == np.uint8
assert tensor.flags.c_contiguous
assert tensor.nbytes == 1_440_000
np.testing.assert_array_equal(tensor[0], non_contiguous)
def test_native_postprocess_uses_608_model_canvas_then_clips_to_raw_raster() -> None:
result = postprocess_native_rf_detr(
_output(),
np.ones((600, 800), dtype=np.bool_),
)
assert tuple(item.label for item in result.detections) == ("person", "dog")
assert result.detections[0].score == pytest.approx(0.8, abs=0.001)
assert result.detections[1].score == pytest.approx(0.75, abs=0.001)
assert result.detections[1].bbox_xyxy == pytest.approx(
(300.0, 228.0, 500.0, 380.0),
abs=0.03,
)
assert dict(result.rejected) == {"non-risk-class": 1}
def test_native_shadow_provider_uses_one_raw_pass_and_preserves_identity() -> None:
backend = _Backend(_output())
provider = NativeRfDetrShadowDetectorProvider(
mask=np.ones((600, 800), dtype=np.bool_),
backend=backend,
clock_ns=iter((10, 30)).__next__,
)
proposals = provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8)))
assert backend.calls == 1
assert tuple(item.semantic_hint for item in proposals) == ("person", "dog")
assert all(item.provider_id == RF_DETR_NATIVE_SHADOW_PROVIDER_ID for item in proposals)
assert all(item.model_id == RF_DETR_NATIVE_SHADOW_MODEL_ID for item in proposals)
assert all(item.preprocess_id == RF_DETR_NATIVE_SHADOW_PREPROCESS_ID for item in proposals)
assert provider.snapshot().completed_frames == 1
assert provider.snapshot().proposal_count == 2
assert provider.snapshot().core_duration_ns == 20
def test_native_shadow_provider_reports_prepare_transport_and_postprocess_timing() -> None:
observed: list[DetectorFrameTiming] = []
provider = NativeRfDetrShadowDetectorProvider(
mask=np.ones((600, 800), dtype=np.bool_),
backend=_Backend(_output()),
clock_ns=iter((10, 20, 50, 70)).__next__,
timing_observer=observed.append,
)
provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8)))
assert [item.to_dict() for item in observed] == [
{
"sequence": 7,
"preprocess_duration_ns": 10,
"inference_transport_duration_ns": 30,
"postprocess_duration_ns": 20,
"total_duration_ns": 60,
}
]
def test_native_shadow_warmup_is_idempotent_and_excluded_from_frame_counts() -> None:
backend = _Backend(_output())
provider = NativeRfDetrShadowDetectorProvider(
mask=np.ones((600, 800), dtype=np.bool_),
backend=backend,
clock_ns=iter((10, 20, 50, 70)).__next__,
)
first = provider.warm_up()
second = provider.warm_up()
assert first is second
assert backend.calls == 1
assert first.total_duration_ns == 60
assert provider.snapshot().input_frames == 0
assert provider.snapshot().completed_frames == 0
def test_native_profile_is_frozen_and_transport_pins_model_version() -> None:
assert RF_DETR_NATIVE_CONFIG.minimum_score == 0.25
with pytest.raises(NativeRfDetrDetectorError, match="cannot be tuned"):
NativeRfDetrConfig(minimum_score=0.5)
backend = TritonNativeRfDetrHttpInferenceBackend("http://127.0.0.1:8100")
try:
assert backend.path == (
"/v2/models/rf_detr_large_native_kb4/versions/1/infer"
)
finally:
backend.close()
with pytest.raises(NativeRfDetrDetectorError, match="explicit HTTP origin"):
TritonNativeRfDetrHttpInferenceBackend("http://user:secret@127.0.0.1:8100")
def test_native_engine_identity_is_pinned() -> None:
assert RF_DETR_NATIVE_ENGINE_SHA256 == (
"b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695"
)
def test_native_shadow_profile_records_failed_legacy_agreement_without_authority() -> None:
profile = json.loads(
(
REPOSITORY_ROOT
/ "config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"
).read_text("utf-8")
)
assert profile["provider_id"] == RF_DETR_NATIVE_SHADOW_PROVIDER_ID
assert profile["model"]["worker_006_rtx4090_tensorrt_11_engine_sha256"] == (
RF_DETR_NATIVE_ENGINE_SHA256
)
assert profile["preprocessing"]["geometric_resampling"] is False
assert profile["preprocessing"]["resize"] is False
assert profile["qualification"]["native_pytorch_tensorrt_parity"]["passed"] is True
assert (
profile["qualification"]["full_ravnoves00_native_vs_legacy_704"]
["legacy_box_agreement_gate_passed"]
is False
)
assert profile["status"]["production_accepted"] is False
assert not any(profile["authority"].values())
assert (
_validate_detector_profile(
REPOSITORY_ROOT
/ "config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json"
)
== "native-kb4"
)