diff --git a/config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json b/config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json new file mode 100644 index 0000000..6a01d18 --- /dev/null +++ b/config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json @@ -0,0 +1,95 @@ +{ + "schema_version": "missioncore.reference-perception-graph-config/v2", + "graph_id": "reference-perception-graph/v2", + "source_profile_id": "m4-ravnoves00-recorded-realtime/v1", + "providers": [ + { + "role": "source", + "provider_id": "ravnoves00-recorded-source/v1", + "version": "1.0.0", + "revision": "m4-ravnoves00-recorded-realtime/v1", + "sha256": "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" + }, + { + "role": "detector", + "provider_id": "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0", + "version": "0.1.0", + "revision": "rf-detr-large-coco-native-kb4-uint8-trt11-fp16-risk-shadow/v0", + "sha256": "398b1102943e704b08a033b1d04b6bdb039ecdd73a2b2e370a0a9cbcaff501ec" + }, + { + "role": "geometry", + "provider_id": "ravnoves00-geometry-association/v1", + "version": "1.0.0", + "revision": "m4-ravnoves00-e29-e32-geometry/v1", + "sha256": "cc666c9389a5e221957faddec89584709b66918d14abaf646f1832e001421999" + }, + { + "role": "temporal", + "provider_id": "bounded-spatial-temporal-layer/v1", + "version": "1.0.0", + "revision": "m4-bounded-temporal-motion/v1", + "sha256": "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39" + }, + { + "role": "motion", + "provider_id": "class-independent-motion-estimator/v1", + "version": "1.0.0", + "revision": "m4-bounded-temporal-motion/v1", + "sha256": "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39" + }, + { + "role": "rolling", + "provider_id": "rolling-local-obstacle-map/v1", + "version": "1.0.0", + "revision": "ravnoves00-rolling-local-obstacle-map/v1", + "sha256": "f7e3315eaf6ffaf3aee1e04913933812092cf82bbcc9984c1a6fa2d9250e6784" + }, + { + "role": "threat", + "provider_id": "dual-evidence-replay-threat/v3", + "version": "3.0.0", + "revision": "m4-ravnoves00-virtual-corridor/v3", + "sha256": "8c3a5aa837da1f028f5998fb504a1381f9b2b68de6420a32160410b6dc0887c7" + } + ], + "queues": [ + { + "stage_id": "detector", + "capacity": 2, + "deadline_ns": 1000000000, + "terminal_timeout_ns": 90000000000 + }, + { + "stage_id": "geometry", + "capacity": 2, + "deadline_ns": 1500000000, + "terminal_timeout_ns": 90000000000 + }, + { + "stage_id": "temporal", + "capacity": 2, + "deadline_ns": 1750000000, + "terminal_timeout_ns": 90000000000 + }, + { + "stage_id": "rolling", + "capacity": 2, + "deadline_ns": 2000000000, + "terminal_timeout_ns": 90000000000 + }, + { + "stage_id": "threat", + "capacity": 2, + "deadline_ns": 2250000000, + "terminal_timeout_ns": 90000000000 + } + ], + "authority": { + "mode": "replay-simulated", + "physical_live": false, + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false + } +} diff --git a/config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json b/config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json new file mode 100644 index 0000000..b6e7d58 --- /dev/null +++ b/config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json @@ -0,0 +1,114 @@ +{ + "schema_version": "missioncore.rf-detr-native-risk-shadow-profile/v0", + "profile_id": "rf-detr-large-coco-native-kb4-uint8-trt11-fp16-risk-shadow/v0", + "provider_id": "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0", + "model": { + "model_id": "rf_detr_large_native_kb4", + "model_version": 1, + "upstream_version": "1.9.4", + "upstream_revision": "9b009fa928d6218320439803d1da01869a85c072", + "checkpoint_sha256": "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38", + "native_core_onnx_sha256": "62e549748a1d17646b90ad06d3ac8a1b79595b7e9270cac4564418f023079176", + "strongly_typed_fp16_onnx_sha256": "00b29fa2ff3d5fca730ebf3b8c33b10e9d690cc97bbbbfa5563d6e0abf210999", + "fused_uint8_onnx_sha256": "acdd01623a00d100331473c0a99eab1e5adf33117cbab900c8ae078edd4aa346", + "worker_006_rtx4090_tensorrt_11_engine_sha256": "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695", + "input": { + "name": "raw_kb4_bgr", + "datatype": "UINT8", + "shape": [1, 600, 800, 3], + "bytes_per_frame": 1440000 + }, + "outputs": [ + {"name": "dets", "datatype": "FP16", "shape": [1, 300, 4]}, + {"name": "labels", "datatype": "FP16", "shape": [1, 300, 91]} + ] + }, + "preprocessing": { + "execution": "single-fused-tensorrt-gpu-graph", + "source_raster": [800, 600], + "source_color": "BGR", + "model_canvas": [800, 608], + "model_color": "RGB", + "valid_fov_mask_sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63", + "valid_fov_fill_value": 114, + "padding_tblr": [0, 8, 0, 0], + "normalization_mean": [0.485, 0.456, 0.406], + "normalization_std": [0.229, 0.224, 0.225], + "resize": false, + "crop": false, + "rectification": false, + "warp": false, + "geometric_resampling": false + }, + "emission": { + "single_inference_per_source_frame": true, + "minimum_score": 0.25, + "maximum_topk_query_class_pairs": 300, + "behavior_relevant_classes": [ + "person", + "bicycle", + "car", + "motorcycle", + "bus", + "truck", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "skateboard" + ], + "geometry_owns_static_occupancy": true, + "unlisted_semantic_classes_emitted": false, + "minimum_box_area_pixels": 64, + "maximum_box_area_fraction": 0.5, + "minimum_valid_fov_fraction": 0.5, + "require_center_inside_valid_fov": true + }, + "qualification": { + "native_pytorch_tensorrt_parity": { + "report_identity_sha256": "215145fed04b43670a71594a7ea6d8f2a676f2ee781eb1d0bfc05bb3ecdbf17c", + "passed": true, + "risk_detection_precision": 0.988700565, + "risk_detection_recall": 0.983146067, + "matched_mean_iou": 0.988242066 + }, + "full_ravnoves00_native_vs_legacy_704": { + "report_identity_sha256": "729bf02b5b52b22d347b3c960f3ac01539ebc76569273bd85560fed8d7b7616b", + "frame_count": 4489, + "native_total_mean_ms": 12.334212, + "native_total_p95_ms": 19.730654, + "legacy_704_total_mean_ms": 33.995721, + "legacy_704_total_p95_ms": 47.432686, + "transport_bytes_reduction_fraction": 0.757877066, + "native_detection_count": 48583, + "legacy_704_detection_count": 51691, + "legacy_box_agreement_recall_iou_at_least_0_5": 0.766593798, + "legacy_box_agreement_gate_passed": false, + "interpretation": "diagnostic-only because legacy 704 geometrically stretches the raw 4:3 raster" + } + }, + "queue": { + "policy": "bounded-latest-wins", + "capacity": 2 + }, + "status": { + "native_tensor_parity_passed": true, + "full_ravnoves00_runtime_gate_passed": true, + "legacy_704_box_agreement_gate_passed": false, + "integrated_world_state_gate_passed": false, + "production_accepted": false + }, + "authority": { + "ground_truth": false, + "candidate_accepted": false, + "commands_enabled": false, + "actuation_allowed": false, + "navigation_or_safety_accepted": false + } +} diff --git a/experiments/perception/run_m48s_reference_graph_shadow_worker.py b/experiments/perception/run_m48s_reference_graph_shadow_worker.py index cd18726..91eab8f 100644 --- a/experiments/perception/run_m48s_reference_graph_shadow_worker.py +++ b/experiments/perception/run_m48s_reference_graph_shadow_worker.py @@ -25,6 +25,7 @@ from k1link.perception.detector import ( DetectorFrameTiming, DetectorProviderSnapshot, DetectorWarmupSnapshot, + NativeRfDetrShadowDetectorProvider, RfDetrShadowDetectorProvider, ) from k1link.perception.geometry import Ravnoves00GeometryAssociationProvider @@ -446,6 +447,7 @@ def main() -> int: all_pipeline_timings: list[dict[str, object]] = [] all_decode_timings: list[DecodedFrameTiming] = [] all_pacing_timings: list[SourcePacingTiming] = [] + detector_provider_id: str | None = None with ( progress.open("x", encoding="utf-8") as progress_stream, frame_ledger.open("x", encoding="utf-8") as frame_ledger_stream, @@ -478,6 +480,11 @@ def main() -> int: maximum_frames=arguments.maximum_frames, source_rate_hz=arguments.source_rate_hz, ) as runtime: + current_detector_provider_id = runtime.graph.detector.provider_id + if detector_provider_id is None: + detector_provider_id = current_detector_provider_id + elif detector_provider_id != current_detector_provider_id: + raise RuntimeError("detector provider identity changed between loops") for stage_id, attribute in ( ("geometry", "geometry"), ("temporal", "temporal"), @@ -504,7 +511,8 @@ def main() -> int: result = runtime.graph.run() loop_completed_ns = time.monotonic_ns() detector_snapshot = cast( - RfDetrShadowDetectorProvider, + RfDetrShadowDetectorProvider + | NativeRfDetrShadowDetectorProvider, runtime.graph.detector, ).snapshot() provider_snapshots = { @@ -582,6 +590,8 @@ def main() -> int: print(json.dumps(progress_row, sort_keys=True), flush=True) completed_ns = time.monotonic_ns() + if detector_provider_id is None: + raise RuntimeError("detector provider identity was not observed") wall_seconds = (completed_ns - started_ns) / 1_000_000_000.0 rss_after_kib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss accounting: Counter[str] = Counter() @@ -674,7 +684,7 @@ def main() -> int: "identity": { "worker_id": "worker-006", "graph_id": "reference-perception-graph/v2", - "detector_provider_id": "triton-rf-detr-large-coco-risk-fp16-shadow/v0", + "detector_provider_id": detector_provider_id, "inputs": _input_digests(paths, arguments.detector_profile), "runtime_artifact_sha256": arguments.runtime_artifact_sha256, "runner_sha256": arguments.runner_sha256, diff --git a/experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1 b/experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1 new file mode 100644 index 0000000..40ca927 --- /dev/null +++ b/experiments/perception/worker/Invoke-M48NNativeReferenceGraph.ps1 @@ -0,0 +1,360 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ReleaseRoot, + [Parameter(Mandatory = $true)] + [string]$CandidateRoot, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedWheelSha256, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9._-]{1,96}$")] + [string]$RunId, + [ValidateRange(1, 4489)] + [int]$MaximumFrames = 300, + [ValidateRange(1.0, 120.0)] + [double]$SourceRateHz = 10.0, + [ValidateRange(0.0, 1.0)] + [double]$MinimumDeliveryRatio = 0.999, + [ValidateRange(0.1, 120.0)] + [double]$MinimumEffectiveWorldStateFps = 9.5, + [ValidateRange(1.0, 10000.0)] + [double]$MaximumWorldStateCompletionP95Ms = 125.0, + [string]$OutputRoot = ( + "D:\NDC_MISSIONCORE\runtime\results\m48n-native-reference-graph-shadow" + ) +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-LastExitCode([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Assert-File([string]$Path, [string]$ExpectedSha256, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ($item.PSIsContainer -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "$Label must be a regular file" + } + if ((Get-Sha256 $item.FullName) -cne $ExpectedSha256) { + throw "$Label SHA-256 changed" + } + return $item.FullName +} + +function Resolve-DDirectory([string]$Path, [string]$Label, [bool]$Create) { + if ($Create -and -not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName +} + +function Convert-ToDockerPath([string]$Path) { return $Path.Replace("\", "/") } + +function Get-Container([string]$Name) { + $rows = @((& docker inspect $Name) | ConvertFrom-Json) + Assert-LastExitCode "Docker inspection for $Name" + if ($rows.Count -ne 1) { throw "Container identity for $Name is not unique" } + return $rows[0] +} + +if ($env:COMPUTERNAME -cne "DESKTOP-OPJ8J04") { + throw "M48N native reference graph is pinned to DESKTOP-OPJ8J04" +} + +$release = Resolve-DDirectory $ReleaseRoot "M48N release root" $false +$candidate = Resolve-DDirectory $CandidateRoot "M48N candidate root" $false +$output = Resolve-DDirectory $OutputRoot "M48N output root" $true +$runOutput = Join-Path $output $RunId +if (Test-Path -LiteralPath $runOutput) { throw "M48N run output already exists" } +$null = New-Item -ItemType Directory -Path $runOutput +$runOutput = Resolve-DDirectory $runOutput "M48N run output" $false + +$wheel = Assert-File ( + Join-Path $release "nodedc_mission_core-0.1.0-py3-none-any.whl" +) $ExpectedWheelSha256 "M48N wheel" +$expectedConfigs = [ordered]@{ + "m48n-rf-detr-native-reference-graph-shadow-v0.json" = ( + "336dccb6b64f4fa5def14aae967fd3280cc1720e93fa3ee21885e0c3304cee4d" + ) + "m4-recorded-realtime-baseline-v1.json" = ( + "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" + ) + "rf-detr-large-native-kb4-risk-shadow-v0.json" = ( + "398b1102943e704b08a033b1d04b6bdb039ecdd73a2b2e370a0a9cbcaff501ec" + ) + "m4-geometry-association-v1.json" = ( + "cc666c9389a5e221957faddec89584709b66918d14abaf646f1832e001421999" + ) + "m4-temporal-motion-v1.json" = ( + "7130eaee24a95c7d888bf7598010e03e129e1c3ac5b34bcd8401015ff4244b39" + ) + "m4-rolling-local-map-v1.json" = ( + "f7e3315eaf6ffaf3aee1e04913933812092cf82bbcc9984c1a6fa2d9250e6784" + ) + "m4-replay-threat-v3.json" = ( + "8c3a5aa837da1f028f5998fb504a1381f9b2b68de6420a32160410b6dc0887c7" + ) +} +foreach ($entry in $expectedConfigs.GetEnumerator()) { + $null = Assert-File (Join-Path $release $entry.Key) $entry.Value ( + "M48N config {0}" -f $entry.Key + ) +} +$runner = Get-Item -LiteralPath ( + Join-Path $release "run_m48s_reference_graph_shadow_worker.py" +) +if ($runner.PSIsContainer -or ($runner.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "M48N graph runner must be a regular file" +} +$runnerSha256 = Get-Sha256 $runner.FullName +$nativeConfig = Assert-File ( + Join-Path $release "rf_detr_large_native_kb4_config.pbtxt" +) "15e100029df92c1390c567517eac6d8bf640591c865bf87a3955289292ba3a22" ( + "native RF-DETR Triton config" +) +$nativeEngine = Assert-File ( + Join-Path $candidate "rf-detr-native-uint8.plan" +) "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695" ( + "native RF-DETR TensorRT engine" +) + +$modelRoot = Join-Path $runOutput "triton-models" +$modelDirectory = Join-Path $modelRoot "rf_detr_large_native_kb4" +$modelVersionDirectory = Join-Path $modelDirectory "1" +$null = New-Item -ItemType Directory -Path $modelVersionDirectory +Copy-Item -LiteralPath $nativeConfig -Destination (Join-Path $modelDirectory "config.pbtxt") +Copy-Item -LiteralPath $nativeEngine -Destination (Join-Path $modelVersionDirectory "model.plan") +if ( + (Get-Sha256 (Join-Path $modelVersionDirectory "model.plan")) -cne + "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695" +) { + throw "staged native RF-DETR TensorRT engine SHA-256 changed" +} + +$source = [ordered]@{ + CameraIndex = ( + "D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d" + + "\input\camera\sensor.camera.right\epoch-1\index.jsonl" + ) + SourcePack = ( + "D:\NDC_MISSIONCORE\runtime\derived" + + "\e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b" + + "\lidar-pack.npz" + ) + LocalSurface = ( + "D:\NDC_MISSIONCORE\runtime\derived" + + "\k1-local-surface-23762244c8bdb97de26fb721ac957d7a00bc9a63571ac4cfa4be19c4effc7d55" + + "\local-surface.npz" + ) + Video = ( + "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs" + + "\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4" + ) + Mask = ( + "D:\NDC_MISSIONCORE\runtime\inputs\e2" + + "\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" + + "\mask.png" + ) +} +foreach ($entry in $source.GetEnumerator()) { + if (-not (Test-Path -LiteralPath $entry.Value -PathType Leaf)) { + throw "M48N source $($entry.Key) is missing" + } +} +if ((Get-Sha256 $source.Video) -cne "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8") { + throw "RAVNOVES00 video SHA-256 changed" +} +if ((Get-Sha256 $source.Mask) -cne "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63") { + throw "valid-FOV mask SHA-256 changed" +} + +$media = Resolve-DDirectory ( + "D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1" +) "PyAV dependency" $false +$opencv = Resolve-DDirectory ( + "D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1\packages" +) "OpenCV dependency" $false +$pillow = Resolve-DDirectory ( + "D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1" +) "Pillow dependency" $false + +$image = ( + "nvcr.io/nvidia/tritonserver:26.06-py3@" + + "sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" +) +& docker image inspect $image *> $null +Assert-LastExitCode "pinned M48N image inspection" +$canonicalTriton = Get-Container "ndc-mission-core-triton" +if (-not $canonicalTriton.State.Running -or $canonicalTriton.State.Health.Status -cne "healthy") { + throw "Canonical Triton must remain healthy during M48N shadow" +} +$canonicalTritonId = [string]$canonicalTriton.Id +$tritonName = "ndc-mission-core-m48n-native-reference-graph-triton" +$graphName = "ndc-mission-core-m48n-native-reference-graph" +foreach ($name in @($tritonName, $graphName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + throw "M48N candidate container $name already exists" + } +} + +try { + & docker create ` + --name $tritonName ` + --label "com.nodedc.product=mission-core" ` + --label "com.nodedc.stack=ndc-mission-core-compute" ` + --label "com.nodedc.role=bounded-native-rf-detr-reference-graph-triton" ` + --label "com.nodedc.managed-by=codex-bounded-experiment" ` + --read-only ` + --security-opt "no-new-privileges:true" ` + --cap-drop ALL ` + --pids-limit 512 ` + --shm-size 1g ` + --gpus all ` + --tmpfs "/tmp:rw,noexec,nosuid,size=2g" ` + --health-cmd "curl --fail --silent http://127.0.0.1:8000/v2/health/ready" ` + --health-interval 5s ` + --health-timeout 3s ` + --health-start-period 20s ` + --health-retries 24 ` + -v ((Convert-ToDockerPath $modelRoot) + ":/models:ro") ` + $image ` + tritonserver ` + --model-repository=/models ` + --model-control-mode=explicit ` + --load-model=rf_detr_large_native_kb4 ` + --disable-auto-complete-config ` + --strict-readiness=true ` + --exit-on-error=true ` + --allow-http=true ` + --allow-grpc=false ` + --allow-metrics=false *> $null + Assert-LastExitCode "M48N Triton creation" + & docker start $tritonName *> $null + Assert-LastExitCode "M48N Triton start" + $ready = $false + foreach ($attempt in 1..60) { + Start-Sleep -Seconds 2 + $candidateContainer = Get-Container $tritonName + if (-not $candidateContainer.State.Running) { + & docker logs $tritonName + throw "M48N Triton stopped during startup" + } + if ($candidateContainer.State.Health.Status -ceq "healthy") { + $ready = $true + break + } + } + if (-not $ready) { throw "M48N Triton did not become healthy" } + if (@((Get-Container $tritonName).HostConfig.PortBindings.PSObject.Properties).Count -ne 0) { + throw "M48N Triton published a host port" + } + + $arguments = @( + "run", "--name", $graphName, + "--label", "com.nodedc.product=mission-core", + "--label", "com.nodedc.stack=ndc-mission-core-compute", + "--label", "com.nodedc.role=bounded-native-rf-detr-reference-graph", + "--label", "com.nodedc.managed-by=codex-bounded-experiment", + "--network", ("container:{0}" -f $tritonName), + "--read-only", + "--security-opt", "no-new-privileges:true", + "--cap-drop", "ALL", + "--pids-limit", "256", + "--gpus", "all", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", ( + "PYTHONPATH=/release/nodedc_mission_core-0.1.0-py3-none-any.whl:" + + "/opt/media:/opt/opencv:/opt/pillow" + ), + "-v", ((Convert-ToDockerPath $release) + ":/release:ro"), + "-v", ((Convert-ToDockerPath $runOutput) + ":/output:rw"), + "-v", ((Convert-ToDockerPath $media) + ":/opt/media:ro"), + "-v", ((Convert-ToDockerPath $opencv) + ":/opt/opencv:ro"), + "-v", ((Convert-ToDockerPath $pillow) + ":/opt/pillow:ro"), + "-v", ((Convert-ToDockerPath $source.CameraIndex) + ":/source/camera-index.jsonl:ro"), + "-v", ((Convert-ToDockerPath $source.SourcePack) + ":/source/source-pack.npz:ro"), + "-v", ((Convert-ToDockerPath $source.LocalSurface) + ":/source/local-surface.npz:ro"), + "-v", ((Convert-ToDockerPath $source.Video) + ":/source/right.mp4:ro"), + "-v", ((Convert-ToDockerPath $source.Mask) + ":/source/mask.png:ro"), + "--entrypoint", "python3", + $image, + "/release/run_m48s_reference_graph_shadow_worker.py", + "--graph-config", "/release/m48n-rf-detr-native-reference-graph-shadow-v0.json", + "--baseline-profile", "/release/m4-recorded-realtime-baseline-v1.json", + "--detector-profile", "/release/rf-detr-large-native-kb4-risk-shadow-v0.json", + "--geometry-profile", "/release/m4-geometry-association-v1.json", + "--temporal-motion-profile", "/release/m4-temporal-motion-v1.json", + "--rolling-map-profile", "/release/m4-rolling-local-map-v1.json", + "--threat-profile", "/release/m4-replay-threat-v3.json", + "--camera-index", "/source/camera-index.jsonl", + "--source-pack", "/source/source-pack.npz", + "--local-surface", "/source/local-surface.npz", + "--video", "/source/right.mp4", + "--valid-fov-mask", "/source/mask.png", + "--triton-origin", "http://127.0.0.1:8000", + "--loops", "1", + "--maximum-frames", ([string]$MaximumFrames), + "--source-rate-hz", ([string]::Format( + [Globalization.CultureInfo]::InvariantCulture, "{0:R}", $SourceRateHz + )), + "--minimum-delivery-ratio", ([string]::Format( + [Globalization.CultureInfo]::InvariantCulture, "{0:R}", $MinimumDeliveryRatio + )), + "--minimum-effective-world-state-fps", ([string]::Format( + [Globalization.CultureInfo]::InvariantCulture, + "{0:R}", + $MinimumEffectiveWorldStateFps + )), + "--maximum-world-state-completion-p95-ms", ([string]::Format( + [Globalization.CultureInfo]::InvariantCulture, + "{0:R}", + $MaximumWorldStateCompletionP95Ms + )), + "--load-purpose", "production-rate", + "--runtime-artifact-sha256", $ExpectedWheelSha256, + "--runner-sha256", $runnerSha256, + "--output", "/output/result.json", + "--progress", "/output/progress.jsonl", + "--frame-ledger", "/output/frames.jsonl" + ) + & docker @arguments + Assert-LastExitCode "M48N native complete reference graph shadow" + foreach ($name in @("result.json", "frames.jsonl", "progress.jsonl")) { + if (-not (Test-Path -LiteralPath (Join-Path $runOutput $name) -PathType Leaf)) { + throw "M48N graph artifact $name was not written" + } + } +} finally { + foreach ($name in @($graphName, $tritonName)) { + if (& docker ps -a --format "{{.Names}}" --filter "name=^/$name$") { + & docker rm -f $name *> $null + } + } + $canonicalAfter = Get-Container "ndc-mission-core-triton" + if ( + $canonicalAfter.Id -cne $canonicalTritonId -or + -not $canonicalAfter.State.Running -or + $canonicalAfter.State.Health.Status -cne "healthy" + ) { + throw "Canonical Triton changed during M48N shadow" + } +} + +Write-Output ("M48N_NATIVE_REFERENCE_GRAPH_RESULT={0}" -f (Join-Path $runOutput "result.json")) +Write-Output "CANONICAL_TRITON_ACTION=none" +Write-Output "PRODUCTION_ACCEPTED=false" diff --git a/src/k1link/perception/detector.py b/src/k1link/perception/detector.py index 1875226..1b1f5e0 100644 --- a/src/k1link/perception/detector.py +++ b/src/k1link/perception/detector.py @@ -14,6 +14,15 @@ from numpy.typing import NDArray from .contracts import BoundingRegion2D, ObjectProposal2D from .providers import SourcePacket +from .rf_detr_native_object_detector import ( + RF_DETR_NATIVE_CONFIG, + RF_DETR_NATIVE_MODEL_ID, + RF_DETR_NATIVE_MODEL_VERSION, + NativeRfDetrConfig, + NativeRfDetrInferenceBackend, + postprocess_native_rf_detr, + prepare_raw_kb4_rf_detr_native, +) from .rf_detr_object_detector import ( RF_DETR_CONFIG, RF_DETR_MODEL_ID, @@ -45,6 +54,15 @@ FROZEN_YOLOX_PREPROCESS_ID: Final = "raw-kb4-valid-fov-letterbox/v1" RF_DETR_SHADOW_PROVIDER_ID: Final = "triton-rf-detr-large-coco-risk-fp16-shadow/v0" RF_DETR_SHADOW_MODEL_ID: Final = f"{RF_DETR_MODEL_ID}:{RF_DETR_MODEL_VERSION}" RF_DETR_SHADOW_PREPROCESS_ID: Final = "raw-kb4-valid-fov-rgb-stretch-imagenet/v0" +RF_DETR_NATIVE_SHADOW_PROVIDER_ID: Final = ( + "triton-rf-detr-large-coco-native-kb4-risk-fp16-shadow/v0" +) +RF_DETR_NATIVE_SHADOW_MODEL_ID: Final = ( + f"{RF_DETR_NATIVE_MODEL_ID}:{RF_DETR_NATIVE_MODEL_VERSION}" +) +RF_DETR_NATIVE_SHADOW_PREPROCESS_ID: Final = ( + "raw-kb4-uint8-fused-mask-rgb-pad8-imagenet-trt/v0" +) class DetectorProviderError(RuntimeError): @@ -414,6 +432,164 @@ def proposals_from_rf_detr_detections( ) +class NativeRfDetrShadowDetectorProvider: + """Emit risk classes from one exact-raster native RF-DETR inference pass.""" + + provider_id: str = RF_DETR_NATIVE_SHADOW_PROVIDER_ID + + def __init__( + self, + *, + mask: NDArray[np.bool_], + backend: NativeRfDetrInferenceBackend, + config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG, + clock_ns: Callable[[], int] = time.perf_counter_ns, + timing_observer: DetectorTimingObserver | None = None, + ) -> None: + if mask.shape != (600, 800) or mask.dtype != np.bool_ or not np.any(mask): + raise DetectorProviderError("native RF-DETR valid-FOV mask is incompatible") + self.mask = np.asarray(mask, dtype=np.bool_) + self.backend = backend + self.config = config + self._clock_ns = clock_ns + self.timing_observer = timing_observer + self._lock = Lock() + self._input_frames = 0 + self._completed_frames = 0 + self._failed_frames = 0 + self._zero_proposal_frames = 0 + self._proposal_count = 0 + self._rejected: Counter[str] = Counter() + self._core_duration_ns = 0 + self._warmup_started = False + self._warmup_snapshot: DetectorWarmupSnapshot | None = None + + def warm_up(self) -> DetectorWarmupSnapshot: + """Prime raw transport and postprocessing before source admission.""" + + with self._lock: + if self._warmup_snapshot is not None: + return self._warmup_snapshot + if self._warmup_started: + raise DetectorProviderError("native RF-DETR warmup is already in progress") + self._warmup_started = True + started_ns = int(self._clock_ns()) + try: + image = np.zeros( + (self.config.source_height, self.config.source_width, 3), + dtype=np.uint8, + ) + tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config) + preprocessed_ns = int(self._clock_ns()) + output = self.backend.infer(tensor) + inferred_ns = int(self._clock_ns()) + postprocess_native_rf_detr(output, self.mask, config=self.config) + completed_ns = int(self._clock_ns()) + except Exception: + with self._lock: + self._warmup_started = False + raise + snapshot = DetectorWarmupSnapshot( + completed=True, + inference_passes=1, + preprocess_duration_ns=max(0, preprocessed_ns - started_ns), + inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns), + postprocess_duration_ns=max(0, completed_ns - inferred_ns), + total_duration_ns=max(0, completed_ns - started_ns), + ) + with self._lock: + self._warmup_snapshot = snapshot + return snapshot + + def detect(self, packet: SourcePacket) -> tuple[ObjectProposal2D, ...]: + payload = packet.image_payload + with self._lock: + self._input_frames += 1 + started_ns = int(self._clock_ns()) + try: + if not isinstance(payload, np.ndarray): + raise DetectorProviderError( + "native RF-DETR requires a decoded BGR image payload" + ) + image = np.asarray(payload) + if image.dtype != np.uint8: + raise DetectorProviderError("decoded BGR image must be uint8") + tensor = prepare_raw_kb4_rf_detr_native(image, config=self.config) + preprocessed_ns = ( + int(self._clock_ns()) if self.timing_observer is not None else started_ns + ) + output = self.backend.infer(tensor) + inferred_ns = ( + int(self._clock_ns()) if self.timing_observer is not None else preprocessed_ns + ) + postprocessed = postprocess_native_rf_detr( + output, + self.mask, + config=self.config, + ) + proposals = proposals_from_native_rf_detr_detections( + packet, + postprocessed.detections, + ) + except Exception: + with self._lock: + self._failed_frames += 1 + self._core_duration_ns += max(0, int(self._clock_ns()) - started_ns) + raise + completed_ns = int(self._clock_ns()) + with self._lock: + self._completed_frames += 1 + self._proposal_count += len(proposals) + self._zero_proposal_frames += not proposals + self._rejected.update(dict(postprocessed.rejected)) + self._core_duration_ns += max(0, completed_ns - started_ns) + if self.timing_observer is not None: + self.timing_observer( + DetectorFrameTiming( + sequence=packet.envelope.sequence, + preprocess_duration_ns=max(0, preprocessed_ns - started_ns), + inference_transport_duration_ns=max(0, inferred_ns - preprocessed_ns), + postprocess_duration_ns=max(0, completed_ns - inferred_ns), + total_duration_ns=max(0, completed_ns - started_ns), + ) + ) + return proposals + + def snapshot(self) -> DetectorProviderSnapshot: + with self._lock: + return DetectorProviderSnapshot( + input_frames=self._input_frames, + completed_frames=self._completed_frames, + failed_frames=self._failed_frames, + zero_proposal_frames=self._zero_proposal_frames, + proposal_count=self._proposal_count, + rejected=tuple(sorted(self._rejected.items())), + core_duration_ns=self._core_duration_ns, + ) + + +def proposals_from_native_rf_detr_detections( + packet: SourcePacket, + detections: tuple[RfDetrDetection, ...], +) -> tuple[ObjectProposal2D, ...]: + envelope = packet.envelope + return tuple( + ObjectProposal2D( + proposal_id=f"proposal-{envelope.sequence}-{index}", + source_id=envelope.source_id, + frame_id=envelope.frame_id, + region=BoundingRegion2D(*detection.bbox_xyxy), + objectness=detection.score, + provider_id=RF_DETR_NATIVE_SHADOW_PROVIDER_ID, + model_id=RF_DETR_NATIVE_SHADOW_MODEL_ID, + preprocess_id=RF_DETR_NATIVE_SHADOW_PREPROCESS_ID, + semantic_hint=detection.label, + provider_tracklet=None, + ) + for index, detection in enumerate(detections) + ) + + __all__ = [ "ALL_COCO_YOLOX_PROVIDER_ID", "FROZEN_YOLOX_MODEL_ID", @@ -422,6 +598,9 @@ __all__ = [ "RF_DETR_SHADOW_MODEL_ID", "RF_DETR_SHADOW_PREPROCESS_ID", "RF_DETR_SHADOW_PROVIDER_ID", + "RF_DETR_NATIVE_SHADOW_MODEL_ID", + "RF_DETR_NATIVE_SHADOW_PREPROCESS_ID", + "RF_DETR_NATIVE_SHADOW_PROVIDER_ID", "DetectorProviderError", "DetectorProviderSnapshot", "DetectorFrameTiming", @@ -429,7 +608,9 @@ __all__ = [ "DetectorWarmupSnapshot", "AllCocoYoloxDetectorProvider", "FrozenYoloxDetectorProvider", + "NativeRfDetrShadowDetectorProvider", "RfDetrShadowDetectorProvider", "proposals_from_detections", + "proposals_from_native_rf_detr_detections", "proposals_from_rf_detr_detections", ] diff --git a/src/k1link/perception/m48s_reference_graph_runtime.py b/src/k1link/perception/m48s_reference_graph_runtime.py index 1346ae8..61db5ff 100644 --- a/src/k1link/perception/m48s_reference_graph_runtime.py +++ b/src/k1link/perception/m48s_reference_graph_runtime.py @@ -8,12 +8,15 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path from threading import Event +from typing import Literal from .baseline import load_m4_baseline from .detector import ( + RF_DETR_NATIVE_SHADOW_PROVIDER_ID, RF_DETR_SHADOW_PROVIDER_ID, DetectorTimingObserver, DetectorWarmupSnapshot, + NativeRfDetrShadowDetectorProvider, RfDetrShadowDetectorProvider, ) from .geometry import ( @@ -41,6 +44,12 @@ from .recorded_source import ( SourcePacingObserver, ) from .reference_graph_runtime import ReferenceGraphRuntimePaths +from .rf_detr_native_object_detector import ( + RF_DETR_NATIVE_ENGINE_SHA256, + RF_DETR_NATIVE_MODEL_ID, + RF_DETR_NATIVE_MODEL_VERSION, + TritonNativeRfDetrHttpInferenceBackend, +) from .rf_detr_object_detector import ( RF_DETR_ENGINE_SHA256, RF_DETR_MODEL_ID, @@ -66,13 +75,18 @@ class M48sReferenceGraphRuntime: """Own one RF-DETR shadow graph and its persistent inference transport.""" graph: ReferencePerceptionGraphV2 - inference_backend: TritonRfDetrHttpInferenceBackend + inference_backend: ( + TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend + ) source_prefetch: PrefetchedRecordedImageDecoder _preparation_stop_event: Event = field(default_factory=Event) def warm_up_detector(self) -> DetectorWarmupSnapshot: detector = self.graph.detector - if not isinstance(detector, RfDetrShadowDetectorProvider): + if not isinstance( + detector, + (RfDetrShadowDetectorProvider, NativeRfDetrShadowDetectorProvider), + ): raise M48sReferenceGraphRuntimeError("RF-DETR runtime detector changed before warmup") return detector.warm_up() @@ -125,7 +139,7 @@ def build_m48s_reference_graph_runtime( ProviderRole.THREAT: paths.threat_profile, } _validate_provider_digests(config, pinned_files) - _validate_detector_profile(detector_profile) + detector_variant = _validate_detector_profile(detector_profile) load_m4_baseline(paths.baseline_profile) geometry_profile = load_geometry_profile(paths.geometry_profile) @@ -161,7 +175,22 @@ def build_m48s_reference_graph_runtime( ) if maximum_frames is not None: source = _LimitedSource(source, maximum_frames) - backend = TritonRfDetrHttpInferenceBackend(triton_origin) + if detector_variant == "legacy-704": + backend: ( + TritonRfDetrHttpInferenceBackend | TritonNativeRfDetrHttpInferenceBackend + ) = TritonRfDetrHttpInferenceBackend(triton_origin) + detector = RfDetrShadowDetectorProvider( + mask=load_valid_fov_mask(paths.valid_fov_mask), + backend=backend, + timing_observer=detector_timing_observer, + ) + else: + backend = TritonNativeRfDetrHttpInferenceBackend(triton_origin) + detector = NativeRfDetrShadowDetectorProvider( + mask=load_valid_fov_mask(paths.valid_fov_mask), + backend=backend, + timing_observer=detector_timing_observer, + ) try: store = RecordedGeometryStore( source_pack_path=paths.source_pack, @@ -175,11 +204,7 @@ def build_m48s_reference_graph_runtime( graph = ReferencePerceptionGraphV2( config=config, source=source, - detector=RfDetrShadowDetectorProvider( - mask=load_valid_fov_mask(paths.valid_fov_mask), - backend=backend, - timing_observer=detector_timing_observer, - ), + detector=detector, geometry=Ravnoves00GeometryAssociationProvider(store=store), temporal=BoundedSpatialTemporalProvider( point_resolver=store, @@ -249,7 +274,7 @@ def _validate_provider_digests( raise M48sReferenceGraphRuntimeError(f"{role.value} provider profile digest changed") -def _validate_detector_profile(path: Path) -> None: +def _validate_detector_profile(path: Path) -> Literal["legacy-704", "native-kb4"]: try: document = json.loads(path.resolve(strict=True).read_text("utf-8")) model = document["model"] @@ -257,24 +282,46 @@ def _validate_detector_profile(path: Path) -> None: authority = document["authority"] except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: raise M48sReferenceGraphRuntimeError("RF-DETR profile is incomplete") from exc - if ( - document.get("schema_version") != "missioncore.rf-detr-risk-shadow-profile/v0" - or document.get("provider_id") != RF_DETR_SHADOW_PROVIDER_ID - or model.get("model_id") != RF_DETR_MODEL_ID - or model.get("model_version") != RF_DETR_MODEL_VERSION - or model.get("worker_006_rtx4090_tensorrt_11_engine_sha256") != RF_DETR_ENGINE_SHA256 - or status.get("detector_load_gate_passed") is not True - or status.get("production_accepted") is not False - or any( - authority.get(key) is not False - for key in ( - "candidate_accepted", - "commands_enabled", - "actuation_allowed", - "navigation_or_safety_accepted", - ) + authority_false = not any( + authority.get(key) is not False + for key in ( + "candidate_accepted", + "commands_enabled", + "actuation_allowed", + "navigation_or_safety_accepted", ) - ): + ) + legacy = ( + document.get("schema_version") == "missioncore.rf-detr-risk-shadow-profile/v0" + and document.get("provider_id") == RF_DETR_SHADOW_PROVIDER_ID + and model.get("model_id") == RF_DETR_MODEL_ID + and model.get("model_version") == RF_DETR_MODEL_VERSION + and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256") + == RF_DETR_ENGINE_SHA256 + and status.get("detector_load_gate_passed") is True + and status.get("production_accepted") is False + and authority_false + ) + native = ( + document.get("schema_version") + == "missioncore.rf-detr-native-risk-shadow-profile/v0" + and document.get("provider_id") == RF_DETR_NATIVE_SHADOW_PROVIDER_ID + and model.get("model_id") == RF_DETR_NATIVE_MODEL_ID + and model.get("model_version") == RF_DETR_NATIVE_MODEL_VERSION + and model.get("worker_006_rtx4090_tensorrt_11_engine_sha256") + == RF_DETR_NATIVE_ENGINE_SHA256 + and status.get("native_tensor_parity_passed") is True + and status.get("full_ravnoves00_runtime_gate_passed") is True + and status.get("legacy_704_box_agreement_gate_passed") is False + and status.get("integrated_world_state_gate_passed") is False + and status.get("production_accepted") is False + and authority_false + ) + if legacy: + return "legacy-704" + if native: + return "native-kb4" + else: raise M48sReferenceGraphRuntimeError("RF-DETR shadow profile identity changed") diff --git a/src/k1link/perception/rf_detr_native_object_detector.py b/src/k1link/perception/rf_detr_native_object_detector.py new file mode 100644 index 0000000..d6b0db8 --- /dev/null +++ b/src/k1link/perception/rf_detr_native_object_detector.py @@ -0,0 +1,364 @@ +"""Native raw-KB4 RF-DETR-L TensorRT transport and postprocessing. + +The TensorRT engine owns valid-FOV masking, BGR-to-RGB conversion, eight +bottom padding rows and ImageNet normalization. The client sends the exact +800x600 UINT8 KB4 raster and performs no geometric resampling. +""" + +from __future__ import annotations + +import http.client +import json +import math +import urllib.parse +from collections import Counter +from dataclasses import dataclass +from typing import Final, Protocol, cast + +import numpy as np +from numpy.typing import NDArray + +from .rf_detr_object_detector import ( + COCO_SPARSE_TO_CONTIGUOUS, + RISK_CLASS_IDS, + RfDetrDetection, + RfDetrPostprocessResult, + RfDetrRawOutput, +) +from .yolox_object_detector import COCO_CLASSES, YOLOX_VALID_FOV_SHA256 + +RF_DETR_NATIVE_MODEL_ID: Final = "rf_detr_large_native_kb4" +RF_DETR_NATIVE_MODEL_VERSION: Final = 1 +RF_DETR_NATIVE_CHECKPOINT_SHA256: Final = ( + "0f4e20e19a99c0f8a62b5685f57f6c8b5c371c59081feda6752a0561a79ccf38" +) +RF_DETR_NATIVE_CORE_ONNX_SHA256: Final = ( + "62e549748a1d17646b90ad06d3ac8a1b79595b7e9270cac4564418f023079176" +) +RF_DETR_NATIVE_FP16_ONNX_SHA256: Final = ( + "00b29fa2ff3d5fca730ebf3b8c33b10e9d690cc97bbbbfa5563d6e0abf210999" +) +RF_DETR_NATIVE_WRAPPED_ONNX_SHA256: Final = ( + "acdd01623a00d100331473c0a99eab1e5adf33117cbab900c8ae078edd4aa346" +) +RF_DETR_NATIVE_ENGINE_SHA256: Final = ( + "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695" +) +RF_DETR_NATIVE_VALID_FOV_SHA256: Final = YOLOX_VALID_FOV_SHA256 + + +class NativeRfDetrDetectorError(RuntimeError): + """The native RF-DETR profile, tensor or response is incompatible.""" + + +@dataclass(frozen=True, slots=True) +class NativeRfDetrConfig: + source_width: int = 800 + source_height: int = 600 + model_width: int = 800 + model_height: int = 608 + bottom_padding_rows: int = 8 + fill_value: int = 114 + minimum_score: float = 0.25 + target_class_ids: tuple[int, ...] = RISK_CLASS_IDS + maximum_detections: int = 300 + minimum_box_area_pixels: float = 64.0 + maximum_box_area_fraction: float = 0.5 + minimum_valid_fov_fraction: float = 0.5 + require_center_inside_valid_fov: bool = True + + def __post_init__(self) -> None: + if ( + self.source_width, + self.source_height, + self.model_width, + self.model_height, + self.bottom_padding_rows, + self.fill_value, + self.minimum_score, + self.target_class_ids, + self.maximum_detections, + self.minimum_box_area_pixels, + self.maximum_box_area_fraction, + self.minimum_valid_fov_fraction, + self.require_center_inside_valid_fov, + ) != ( + 800, + 600, + 800, + 608, + 8, + 114, + 0.25, + RISK_CLASS_IDS, + 300, + 64.0, + 0.5, + 0.5, + True, + ): + raise NativeRfDetrDetectorError( + "native RF-DETR shadow profile cannot be tuned in place" + ) + + +RF_DETR_NATIVE_CONFIG: Final = NativeRfDetrConfig() + + +class NativeRfDetrInferenceBackend(Protocol): + def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput: ... + + +class TritonNativeRfDetrHttpInferenceBackend: + """Persistent Triton V2 HTTP transport for exact raw UINT8 KB4 frames.""" + + def __init__(self, endpoint: str, *, timeout_seconds: float = 60.0) -> None: + parsed = urllib.parse.urlsplit(endpoint) + if ( + parsed.scheme != "http" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise NativeRfDetrDetectorError( + "Triton endpoint must be an explicit HTTP origin" + ) + if not math.isfinite(timeout_seconds) or timeout_seconds <= 0: + raise NativeRfDetrDetectorError("Triton timeout must be positive") + self.path = ( + f"{parsed.path.rstrip('/')}/v2/models/{RF_DETR_NATIVE_MODEL_ID}" + f"/versions/{RF_DETR_NATIVE_MODEL_VERSION}/infer" + ) + self.connection = http.client.HTTPConnection( + parsed.hostname, + parsed.port or 80, + timeout=timeout_seconds, + ) + + def close(self) -> None: + self.connection.close() + + def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput: + contiguous = np.ascontiguousarray(tensor, dtype=np.uint8) + if contiguous.shape != (1, 600, 800, 3): + raise NativeRfDetrDetectorError( + "Triton native RF-DETR input tensor is incompatible" + ) + binary = contiguous.tobytes() + header = { + "inputs": [ + { + "name": "raw_kb4_bgr", + "shape": [1, 600, 800, 3], + "datatype": "UINT8", + "parameters": {"binary_data_size": len(binary)}, + } + ], + "outputs": [ + {"name": "dets", "parameters": {"binary_data": True}}, + {"name": "labels", "parameters": {"binary_data": True}}, + ], + } + encoded = json.dumps(header, sort_keys=True, separators=(",", ":")).encode() + self.connection.request( + "POST", + self.path, + body=encoded + binary, + headers={ + "Content-Type": "application/octet-stream", + "Inference-Header-Content-Length": str(len(encoded)), + }, + ) + response = self.connection.getresponse() + payload = response.read() + if response.status != 200: + raise NativeRfDetrDetectorError( + f"Triton native RF-DETR inference failed with HTTP {response.status}" + ) + header_value = response.getheader("Inference-Header-Content-Length") + try: + header_length = int(header_value or "") + descriptor = json.loads(payload[:header_length]) + outputs = descriptor["outputs"] + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise NativeRfDetrDetectorError( + "Triton native RF-DETR output descriptor is invalid" + ) from exc + if not isinstance(outputs, list) or len(outputs) != 2: + raise NativeRfDetrDetectorError( + "Triton native RF-DETR output count changed" + ) + offset = header_length + arrays: dict[str, NDArray[np.float16]] = {} + for output, expected_name, expected_shape in zip( + outputs, + ("dets", "labels"), + ((1, 300, 4), (1, 300, 91)), + strict=True, + ): + try: + name = output["name"] + datatype = output["datatype"] + shape = tuple(int(value) for value in output["shape"]) + byte_length = int(output["parameters"]["binary_data_size"]) + except (KeyError, TypeError, ValueError) as exc: + raise NativeRfDetrDetectorError( + "Triton native RF-DETR output descriptor is incomplete" + ) from exc + expected_bytes = math.prod(expected_shape) * np.dtype(" len(payload) + ): + raise NativeRfDetrDetectorError( + "Triton native RF-DETR output identity changed" + ) + array = np.frombuffer(payload[offset : offset + byte_length], dtype=" NDArray[np.uint8]: + """Expose the exact raw KB4 raster as UINT8 NHWC without image transforms.""" + + if image_bgr.shape != (config.source_height, config.source_width, 3): + raise NativeRfDetrDetectorError("raw KB4 image raster changed") + if image_bgr.dtype != np.uint8: + raise NativeRfDetrDetectorError("raw KB4 image must be uint8") + return np.ascontiguousarray(image_bgr[None], dtype=np.uint8) + + +def postprocess_native_rf_detr( + output: RfDetrRawOutput, + mask: NDArray[np.bool_], + *, + config: NativeRfDetrConfig = RF_DETR_NATIVE_CONFIG, +) -> RfDetrPostprocessResult: + if output.boxes.shape != (1, 300, 4) or output.logits.shape != (1, 300, 91): + raise NativeRfDetrDetectorError("native RF-DETR output shapes are incompatible") + if output.boxes.dtype != np.float16 or output.logits.dtype != np.float16: + raise NativeRfDetrDetectorError("native RF-DETR output types are incompatible") + if not np.isfinite(output.boxes).all() or not np.isfinite(output.logits).all(): + raise NativeRfDetrDetectorError("native RF-DETR output contains non-finite values") + if mask.shape != (config.source_height, config.source_width) or mask.dtype != np.bool_: + raise NativeRfDetrDetectorError("valid-FOV mask is incompatible") + logits = output.logits[0].astype(np.float32) + probabilities = 1.0 / (1.0 + np.exp(-np.clip(logits, -80.0, 80.0))) + flattened = probabilities.reshape(-1) + topk = np.argsort(-flattened, kind="stable")[: config.maximum_detections] + integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1) + rejected: Counter[str] = Counter() + result: list[RfDetrDetection] = [] + for flat_index in topk: + score = float(flattened[flat_index]) + if score <= config.minimum_score: + continue + query_index = int(flat_index // output.logits.shape[2]) + sparse_class_id = int(flat_index % output.logits.shape[2]) + class_id = COCO_SPARSE_TO_CONTIGUOUS.get(sparse_class_id) + if class_id is None: + rejected["unmapped-class-slot"] += 1 + continue + if class_id not in config.target_class_ids: + rejected["non-risk-class"] += 1 + continue + center_x, center_y, box_width, box_height = ( + float(value) for value in output.boxes[0, query_index].astype(np.float32) + ) + box = np.asarray( + ( + (center_x - box_width / 2.0) * config.model_width, + (center_y - box_height / 2.0) * config.model_height, + (center_x + box_width / 2.0) * config.model_width, + (center_y + box_height / 2.0) * config.model_height, + ), + dtype=np.float32, + ) + box[[0, 2]] = np.clip(box[[0, 2]], 0, config.source_width) + box[[1, 3]] = np.clip(box[[1, 3]], 0, config.source_height) + fraction, center_inside, area = _valid_fraction(box, integral) + if area < config.minimum_box_area_pixels: + rejected["small-box"] += 1 + continue + if area / (config.source_width * config.source_height) > ( + config.maximum_box_area_fraction + ): + rejected["large-box"] += 1 + continue + if fraction < config.minimum_valid_fov_fraction: + rejected["outside-valid-fov"] += 1 + continue + if config.require_center_inside_valid_fov and not center_inside: + rejected["center-outside-valid-fov"] += 1 + continue + result.append( + RfDetrDetection( + class_id=class_id, + label=COCO_CLASSES[class_id], + score=round(score, 9), + bbox_xyxy=cast( + tuple[float, float, float, float], + tuple(round(float(value), 6) for value in box), + ), + valid_fov_fraction=round(fraction, 6), + ) + ) + result.sort(key=lambda item: (-item.score, item.class_id)) + return RfDetrPostprocessResult(tuple(result), tuple(sorted(rejected.items()))) + + +def _valid_fraction( + box: NDArray[np.float32], integral: NDArray[np.int64] +) -> tuple[float, bool, float]: + height = integral.shape[0] - 1 + width = integral.shape[1] - 1 + x1 = int(np.clip(math.floor(float(box[0])), 0, width)) + y1 = int(np.clip(math.floor(float(box[1])), 0, height)) + x2 = int(np.clip(math.ceil(float(box[2])), 0, width)) + y2 = int(np.clip(math.ceil(float(box[3])), 0, height)) + area = float(max(0, x2 - x1) * max(0, y2 - y1)) + if area <= 0: + return 0.0, False, 0.0 + inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[y1, x1] + center_x = int(np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1)) + center_y = int(np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1)) + center_inside = bool( + integral[center_y + 1, center_x + 1] + - integral[center_y, center_x + 1] + - integral[center_y + 1, center_x] + + integral[center_y, center_x] + ) + return float(inside) / area, center_inside, area + + +__all__ = [ + "RF_DETR_NATIVE_CHECKPOINT_SHA256", + "RF_DETR_NATIVE_CONFIG", + "RF_DETR_NATIVE_CORE_ONNX_SHA256", + "RF_DETR_NATIVE_ENGINE_SHA256", + "RF_DETR_NATIVE_FP16_ONNX_SHA256", + "RF_DETR_NATIVE_MODEL_ID", + "RF_DETR_NATIVE_MODEL_VERSION", + "RF_DETR_NATIVE_VALID_FOV_SHA256", + "RF_DETR_NATIVE_WRAPPED_ONNX_SHA256", + "NativeRfDetrConfig", + "NativeRfDetrDetectorError", + "NativeRfDetrInferenceBackend", + "TritonNativeRfDetrHttpInferenceBackend", + "postprocess_native_rf_detr", + "prepare_raw_kb4_rf_detr_native", +] diff --git a/tests/test_m48s_reference_graph_shadow.py b/tests/test_m48s_reference_graph_shadow.py index 761d056..cfb0e4d 100644 --- a/tests/test_m48s_reference_graph_shadow.py +++ b/tests/test_m48s_reference_graph_shadow.py @@ -4,7 +4,10 @@ import hashlib import json from pathlib import Path -from k1link.perception.detector import RF_DETR_SHADOW_PROVIDER_ID +from k1link.perception.detector import ( + RF_DETR_NATIVE_SHADOW_PROVIDER_ID, + RF_DETR_SHADOW_PROVIDER_ID, +) from k1link.perception.m48s_advisory import ( AdvisoryFamily, advisory_policy_matrix, @@ -17,6 +20,10 @@ GRAPH_CONFIG = ( REPOSITORY_ROOT / "config/perception/m48s-rf-detr-reference-graph-shadow-v0.json" ) +NATIVE_GRAPH_CONFIG = ( + REPOSITORY_ROOT + / "config/perception/m48n-rf-detr-native-reference-graph-shadow-v0.json" +) def test_m48s_reference_graph_replaces_only_the_detector_pin() -> None: @@ -64,6 +71,51 @@ def test_m48s_reference_graph_pins_every_profile_digest() -> None: assert pins[role].sha256 == hashlib.sha256(payload).hexdigest() +def test_m48n_native_reference_graph_replaces_only_the_detector_pin() -> None: + native = ReferencePerceptionGraphConfigV2.from_dict( + json.loads(NATIVE_GRAPH_CONFIG.read_text("utf-8")) + ) + legacy = ReferencePerceptionGraphConfigV2.from_dict( + json.loads(GRAPH_CONFIG.read_text("utf-8")) + ) + native_pins = {item.role: item for item in native.providers} + legacy_pins = {item.role: item for item in legacy.providers} + + assert native.graph_id == legacy.graph_id == "reference-perception-graph/v2" + assert native.source_profile_id == legacy.source_profile_id + assert native.queues == legacy.queues + assert native.authority == legacy.authority + assert ( + native_pins[ProviderRole.DETECTOR].provider_id + == RF_DETR_NATIVE_SHADOW_PROVIDER_ID + ) + assert all( + native_pins[role] == legacy_pins[role] + for role in ProviderRole + if role is not ProviderRole.DETECTOR + ) + + +def test_m48n_native_reference_graph_pins_every_profile_digest() -> None: + config = ReferencePerceptionGraphConfigV2.from_dict( + json.loads(NATIVE_GRAPH_CONFIG.read_text("utf-8")) + ) + paths = { + ProviderRole.SOURCE: "m4-recorded-realtime-baseline-v1.json", + ProviderRole.DETECTOR: "rf-detr-large-native-kb4-risk-shadow-v0.json", + ProviderRole.GEOMETRY: "m4-geometry-association-v1.json", + ProviderRole.TEMPORAL: "m4-temporal-motion-v1.json", + ProviderRole.MOTION: "m4-temporal-motion-v1.json", + ProviderRole.ROLLING: "m4-rolling-local-map-v1.json", + ProviderRole.THREAT: "m4-replay-threat-v3.json", + } + pins = {item.role: item for item in config.providers} + + for role, name in paths.items(): + payload = (REPOSITORY_ROOT / "config/perception" / name).read_bytes() + assert pins[role].sha256 == hashlib.sha256(payload).hexdigest() + + def test_m48s_advisory_policy_is_bounded_distinct_and_commandless() -> None: matrix = advisory_policy_matrix() diff --git a/tests/test_rf_detr_native_detector_provider.py b/tests/test_rf_detr_native_detector_provider.py new file mode 100644 index 0000000..db6c1bd --- /dev/null +++ b/tests/test_rf_detr_native_detector_provider.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path + +import numpy as np +import pytest +from numpy.typing import NDArray + +from k1link.perception.contracts import ( + ClockBasis, + ModalityOutcome, + ModalityStatus, + SourceEnvelope, + TimestampBundle, +) +from k1link.perception.detector import ( + RF_DETR_NATIVE_SHADOW_MODEL_ID, + RF_DETR_NATIVE_SHADOW_PREPROCESS_ID, + RF_DETR_NATIVE_SHADOW_PROVIDER_ID, + DetectorFrameTiming, + NativeRfDetrShadowDetectorProvider, +) +from k1link.perception.m48s_reference_graph_runtime import _validate_detector_profile +from k1link.perception.providers import SourcePacket +from k1link.perception.rf_detr_native_object_detector import ( + RF_DETR_NATIVE_CONFIG, + RF_DETR_NATIVE_ENGINE_SHA256, + NativeRfDetrConfig, + NativeRfDetrDetectorError, + TritonNativeRfDetrHttpInferenceBackend, + postprocess_native_rf_detr, + prepare_raw_kb4_rf_detr_native, +) +from k1link.perception.rf_detr_object_detector import RfDetrRawOutput + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _status() -> ModalityStatus: + return ModalityStatus(True, ModalityOutcome.AVAILABLE, "test-available") + + +def _packet(sequence: int, image: object) -> SourcePacket: + return SourcePacket( + envelope=SourceEnvelope( + source_id="RAVNOVES00", + session_id="20260720T065719Z_viewer_live", + frame_id=f"frame-{sequence:06d}", + sequence=sequence, + timestamps=TimestampBundle( + utc_ns=1_000 + sequence, + monotonic_ns=2_000 + sequence, + source_ns=3_000 + sequence, + clock_basis=ClockBasis.RECORDED_HOST, + ), + source_age_ns=0, + binding_reason="test-recorded-source", + calibration_id="camera-1-kb4-test", + representation_id="registered-map-increment-v1", + image=_status(), + registered_point_increment=_status(), + pose=_status(), + ), + image_payload=image, + registered_point_increment_payload=("points", sequence), + pose_payload=("pose", sequence), + ) + + +def _output() -> RfDetrRawOutput: + boxes = np.zeros((1, 300, 4), dtype=np.float16) + logits = np.full((1, 300, 91), -20.0, dtype=np.float16) + boxes[0, 0] = (0.5, 0.5, 0.25, 0.25) + logits[0, 0, 18] = np.float16(math.log(3.0)) # dog, score 0.75 + boxes[0, 1] = (0.25, 0.25, 0.1, 0.2) + logits[0, 1, 1] = np.float16(math.log(4.0)) # person, score 0.80 + boxes[0, 2] = (0.75, 0.25, 0.1, 0.2) + logits[0, 2, 62] = np.float16(math.log(9.0)) # chair, non-risk + return RfDetrRawOutput(boxes=boxes, logits=logits) + + +class _Backend: + def __init__(self, output: RfDetrRawOutput) -> None: + self.output = output + self.calls = 0 + + def infer(self, tensor: NDArray[np.uint8]) -> RfDetrRawOutput: + assert tensor.shape == (1, 600, 800, 3) + assert tensor.dtype == np.uint8 + assert tensor.flags.c_contiguous + self.calls += 1 + return self.output + + +def test_native_prepare_preserves_every_raw_pixel_without_geometric_transform() -> None: + raster = np.arange(600 * 800 * 3, dtype=np.uint8).reshape(600, 800, 3) + non_contiguous = raster[:, ::-1] + + tensor = prepare_raw_kb4_rf_detr_native(non_contiguous) + + assert tensor.shape == (1, 600, 800, 3) + assert tensor.dtype == np.uint8 + assert tensor.flags.c_contiguous + assert tensor.nbytes == 1_440_000 + np.testing.assert_array_equal(tensor[0], non_contiguous) + + +def test_native_postprocess_uses_608_model_canvas_then_clips_to_raw_raster() -> None: + result = postprocess_native_rf_detr( + _output(), + np.ones((600, 800), dtype=np.bool_), + ) + + assert tuple(item.label for item in result.detections) == ("person", "dog") + assert result.detections[0].score == pytest.approx(0.8, abs=0.001) + assert result.detections[1].score == pytest.approx(0.75, abs=0.001) + assert result.detections[1].bbox_xyxy == pytest.approx( + (300.0, 228.0, 500.0, 380.0), + abs=0.03, + ) + assert dict(result.rejected) == {"non-risk-class": 1} + + +def test_native_shadow_provider_uses_one_raw_pass_and_preserves_identity() -> None: + backend = _Backend(_output()) + provider = NativeRfDetrShadowDetectorProvider( + mask=np.ones((600, 800), dtype=np.bool_), + backend=backend, + clock_ns=iter((10, 30)).__next__, + ) + + proposals = provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8))) + + assert backend.calls == 1 + assert tuple(item.semantic_hint for item in proposals) == ("person", "dog") + assert all(item.provider_id == RF_DETR_NATIVE_SHADOW_PROVIDER_ID for item in proposals) + assert all(item.model_id == RF_DETR_NATIVE_SHADOW_MODEL_ID for item in proposals) + assert all(item.preprocess_id == RF_DETR_NATIVE_SHADOW_PREPROCESS_ID for item in proposals) + assert provider.snapshot().completed_frames == 1 + assert provider.snapshot().proposal_count == 2 + assert provider.snapshot().core_duration_ns == 20 + + +def test_native_shadow_provider_reports_prepare_transport_and_postprocess_timing() -> None: + observed: list[DetectorFrameTiming] = [] + provider = NativeRfDetrShadowDetectorProvider( + mask=np.ones((600, 800), dtype=np.bool_), + backend=_Backend(_output()), + clock_ns=iter((10, 20, 50, 70)).__next__, + timing_observer=observed.append, + ) + + provider.detect(_packet(7, np.zeros((600, 800, 3), dtype=np.uint8))) + + assert [item.to_dict() for item in observed] == [ + { + "sequence": 7, + "preprocess_duration_ns": 10, + "inference_transport_duration_ns": 30, + "postprocess_duration_ns": 20, + "total_duration_ns": 60, + } + ] + + +def test_native_shadow_warmup_is_idempotent_and_excluded_from_frame_counts() -> None: + backend = _Backend(_output()) + provider = NativeRfDetrShadowDetectorProvider( + mask=np.ones((600, 800), dtype=np.bool_), + backend=backend, + clock_ns=iter((10, 20, 50, 70)).__next__, + ) + + first = provider.warm_up() + second = provider.warm_up() + + assert first is second + assert backend.calls == 1 + assert first.total_duration_ns == 60 + assert provider.snapshot().input_frames == 0 + assert provider.snapshot().completed_frames == 0 + + +def test_native_profile_is_frozen_and_transport_pins_model_version() -> None: + assert RF_DETR_NATIVE_CONFIG.minimum_score == 0.25 + with pytest.raises(NativeRfDetrDetectorError, match="cannot be tuned"): + NativeRfDetrConfig(minimum_score=0.5) + + backend = TritonNativeRfDetrHttpInferenceBackend("http://127.0.0.1:8100") + try: + assert backend.path == ( + "/v2/models/rf_detr_large_native_kb4/versions/1/infer" + ) + finally: + backend.close() + with pytest.raises(NativeRfDetrDetectorError, match="explicit HTTP origin"): + TritonNativeRfDetrHttpInferenceBackend("http://user:secret@127.0.0.1:8100") + + +def test_native_engine_identity_is_pinned() -> None: + assert RF_DETR_NATIVE_ENGINE_SHA256 == ( + "b8a40b3580edff001ec9680de68707242294ff590ab296000fae371f1083f695" + ) + + +def test_native_shadow_profile_records_failed_legacy_agreement_without_authority() -> None: + profile = json.loads( + ( + REPOSITORY_ROOT + / "config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json" + ).read_text("utf-8") + ) + + assert profile["provider_id"] == RF_DETR_NATIVE_SHADOW_PROVIDER_ID + assert profile["model"]["worker_006_rtx4090_tensorrt_11_engine_sha256"] == ( + RF_DETR_NATIVE_ENGINE_SHA256 + ) + assert profile["preprocessing"]["geometric_resampling"] is False + assert profile["preprocessing"]["resize"] is False + assert profile["qualification"]["native_pytorch_tensorrt_parity"]["passed"] is True + assert ( + profile["qualification"]["full_ravnoves00_native_vs_legacy_704"] + ["legacy_box_agreement_gate_passed"] + is False + ) + assert profile["status"]["production_accepted"] is False + assert not any(profile["authority"].values()) + assert ( + _validate_detector_profile( + REPOSITORY_ROOT + / "config/perception/rf-detr-large-native-kb4-risk-shadow-v0.json" + ) + == "native-kb4" + )