feat(perception): add recorded replay maturation labs
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l32_pointpillars_camera_review import (
|
||||
build_l32_pointpillars_camera_review,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
result = build_l32_pointpillars_camera_review(
|
||||
l31_result_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/l3/pointpillars-ravnoves"
|
||||
/ (
|
||||
"l31-pointpillars-ravnoves-"
|
||||
"80a9715f64ea397222fbcfd700803f9152009e3751caf87e2e9dc5ec6fc01b72"
|
||||
)
|
||||
),
|
||||
e10_pack_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/e10/lidar-packs"
|
||||
/ "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b"
|
||||
),
|
||||
camera_job_root=(
|
||||
repository
|
||||
/ ".runtime/compute-jobs"
|
||||
/ "recorded-camera-602ac89026ed12978619801d"
|
||||
),
|
||||
ffmpeg_path=Path("/opt/homebrew/bin/ffmpeg"),
|
||||
output_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/l3/pointpillars-camera-review"
|
||||
),
|
||||
)
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l33_camera_first_admission import (
|
||||
RAVNOVES00_ADMITTED_WORLD_STATE_RESULT_ID,
|
||||
)
|
||||
from k1link.compute.l33_camera_first_detector_review import (
|
||||
build_l33_camera_first_detector_review,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
result = build_l33_camera_first_detector_review(
|
||||
l32_result_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/l3/pointpillars-camera-review"
|
||||
/ (
|
||||
"l32-pointpillars-camera-review-"
|
||||
"40eca128ea9525e8e8c22bd3e981e40d5b66d2869c2e92636ddbf707ae44127a"
|
||||
)
|
||||
),
|
||||
e26_result_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/e10/worker-results"
|
||||
/ (
|
||||
"e10-integrated-perception-"
|
||||
"459aac93918d8f6414b342986ccc6968fefcef6c1f3a78a5254df0b565255ad2"
|
||||
)
|
||||
),
|
||||
e29_result_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/e29/results"
|
||||
/ (
|
||||
"e29-camera-geometry-"
|
||||
"421a9d930638bef12cd5eb10979a477917fa4a389e655ed95f73ba4bd62e13dc"
|
||||
)
|
||||
),
|
||||
e10_pack_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/e10/lidar-packs"
|
||||
/ "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b"
|
||||
),
|
||||
output_root=(repository / ".runtime/compute-experiments/l3/camera-first-detector-review"),
|
||||
rectified_world_state_root=(
|
||||
repository
|
||||
/ ".runtime/compute-experiments/l3/rectified-camera-world-state"
|
||||
/ RAVNOVES00_ADMITTED_WORLD_STATE_RESULT_ID
|
||||
),
|
||||
)
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46e-ready-stack-profile/v1",
|
||||
"profile_id": "e46e-deepstream-trafficcamnet-rtdetr-nvdcf/v1",
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-camera-602ac89026ed12978619801d",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"segment_count": 4489,
|
||||
"stream_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"archive_index_sha256": "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14",
|
||||
"archive_summary_sha256": "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181"
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:10eca409b3894e91c1bac915c9f1346307e56695e552487cbe8cf2f58a3f998f",
|
||||
"container_platform": "linux/amd64",
|
||||
"deepstream_version": "9.1",
|
||||
"network_during_inference": "none"
|
||||
},
|
||||
"detector": {
|
||||
"name": "NVIDIA TrafficCamNet Transformer Lite",
|
||||
"architecture": "RT-DETR ResNet50",
|
||||
"version": "deployable_resnet50_v2.0",
|
||||
"precision": "FP16",
|
||||
"input_shape": [
|
||||
3,
|
||||
544,
|
||||
960
|
||||
],
|
||||
"model_file": "resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"model_sha256": "545a447b913d54eee476381436ebea4ad2aa876cfbe0a6d9f3b0302f08a7415d",
|
||||
"model_url": "https://api.ngc.nvidia.com/v2/models/nvidia/tao/trafficcamnet_transformer_lite/versions/deployable_resnet50_v2.0/files/resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"labels": [
|
||||
"background",
|
||||
"bicycle",
|
||||
"car",
|
||||
"person",
|
||||
"road_sign"
|
||||
],
|
||||
"pre_cluster_threshold": 0.5,
|
||||
"custom_postprocessing": false
|
||||
},
|
||||
"parser": {
|
||||
"name": "NVIDIA DeepStream TAO custom bounding-box parser",
|
||||
"repository": "https://github.com/NVIDIA/DeepStream.git",
|
||||
"commit": "581889df47d6181110c758c10b872ca833a835e3",
|
||||
"source_path": "src/apps/tao_apps/post_processor",
|
||||
"development_image": "nvcr.io/nvidia/deepstream:9.1-triton-multiarch@sha256:fd31f5b44ababdbdee8cd397a375e888191b49e402ac237254a4cdc239130f5b",
|
||||
"cuda_version": "13.2",
|
||||
"symbol": "NvDsInferParseCustomDDETRTAO",
|
||||
"library_file": "libnvds_infercustomparser_tao.so",
|
||||
"library_sha256": "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee",
|
||||
"custom_mission_core_logic": false,
|
||||
"source_files": [
|
||||
{
|
||||
"path": "Makefile",
|
||||
"sha256": "0265f470354e60c6d719bde68c7b74b1879eed9b4552b8fe7b39416af5ce6835"
|
||||
},
|
||||
{
|
||||
"path": "debug_logger_raii.cpp",
|
||||
"sha256": "1d388509e1ff9008de6ccd6451db9c6433273ed78a94e95a1df84585b8dc2915"
|
||||
},
|
||||
{
|
||||
"path": "debug_logger_raii.hpp",
|
||||
"sha256": "6efdce1874468848664a18ceb613f2384b8c079cb12baa888a433d6c0f81b7ec"
|
||||
},
|
||||
{
|
||||
"path": "debug_logger_tensor.hpp",
|
||||
"sha256": "c9999fcf92536bbb36498ddd4485f213fc2f5ddc70d24f8d408195a48680b94f"
|
||||
},
|
||||
{
|
||||
"path": "nvdsinfer_custombboxparser_tao.cpp",
|
||||
"sha256": "1794e3ee5152f25eff31454c6181368676f6659c68fc25b4b1933f6cbb63158b"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"library": "libnvds_nvmultiobjecttracker.so",
|
||||
"configuration": "config_tracker_NvDCF_perf.yml",
|
||||
"past_frame_output": false,
|
||||
"custom_association": false,
|
||||
"custom_hold_or_stitch": false
|
||||
},
|
||||
"output": {
|
||||
"frame_width": 800,
|
||||
"frame_height": 600,
|
||||
"overlay_codec": "H.264",
|
||||
"overlay_container": "MP4",
|
||||
"tracked_frames": "tracked-frames.jsonl"
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46f-dashcam-bakeoff-profile/v1",
|
||||
"profile_id": "e46f-deepstream-dashcamnet-detectnet-v2-nvdcf/v1",
|
||||
"comparison_contract": {
|
||||
"baseline_result_id": "e46e-ready-stack-d51fd744a86b0effa8685c7aa86d14dfd1b12e97bc8d68d0f53f467237b976bf",
|
||||
"controlled_change": "detector-only",
|
||||
"held_constant": [
|
||||
"recorded RIGHT source identity",
|
||||
"DeepStream 9.1 container digest",
|
||||
"FP16 precision",
|
||||
"NVIDIA NvDCF performance configuration",
|
||||
"800x600 output plane",
|
||||
"full 4489-frame replay"
|
||||
]
|
||||
},
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-camera-602ac89026ed12978619801d",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"segment_count": 4489,
|
||||
"stream_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"archive_index_sha256": "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14",
|
||||
"archive_summary_sha256": "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181"
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:10eca409b3894e91c1bac915c9f1346307e56695e552487cbe8cf2f58a3f998f",
|
||||
"container_platform": "linux/amd64",
|
||||
"deepstream_version": "9.1",
|
||||
"network_during_inference": "none"
|
||||
},
|
||||
"detector": {
|
||||
"name": "NVIDIA DashCamNet",
|
||||
"architecture": "DetectNet_v2 ResNet18",
|
||||
"version": "pruned_onnx_v1.0.4",
|
||||
"precision": "FP16",
|
||||
"intended_viewpoint": "moving vehicle dashcam",
|
||||
"input_shape": [
|
||||
3,
|
||||
544,
|
||||
960
|
||||
],
|
||||
"model_file": "resnet18_dashcamnet_pruned.onnx",
|
||||
"model_sha256": "d92f97bd840b68850a33c83e480fc6bd83b3097ff4cb517e0ea12671046bab7d",
|
||||
"model_url": "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_onnx_v1.0.4/files/resnet18_dashcamnet_pruned.onnx",
|
||||
"model_card_url": "https://catalog.ngc.nvidia.com/orgs/nvidia/tao/models/dashcamnet/-",
|
||||
"labels": [
|
||||
"car",
|
||||
"bicycle",
|
||||
"person",
|
||||
"road_sign"
|
||||
],
|
||||
"custom_postprocessing": false,
|
||||
"qualification": {
|
||||
"status": "passed-real-route-tensor-smoke",
|
||||
"method": "ONNX Runtime RGB NCHW 1/255 on immutable RIGHT frames 1248 and 3027",
|
||||
"observed_active_channels": [
|
||||
"car",
|
||||
"person"
|
||||
],
|
||||
"rejected_newer_artifact": {
|
||||
"version": "pruned_onnx_v1.0.5",
|
||||
"model_sha256": "4498f7e86e1113ef918462151ebef187cb9e6451dce4e57bd8e5484e1dd4aabb",
|
||||
"reason": "only confidence channel 2 was non-zero on both immutable qualification frames"
|
||||
}
|
||||
}
|
||||
},
|
||||
"postprocessor": {
|
||||
"name": "NVIDIA DeepStream built-in DetectNet_v2 parser and NMS",
|
||||
"implementation": "built into pinned DeepStream container",
|
||||
"cluster_mode": "NMS",
|
||||
"nms_iou_threshold": 0.5,
|
||||
"pre_cluster_threshold": 0.2,
|
||||
"car_pre_cluster_threshold": 0.4,
|
||||
"topk": 20,
|
||||
"reference_repository": "https://github.com/NVIDIA-AI-IOT/deepstream_reference_apps.git",
|
||||
"reference_commit": "2bda7c6e14effac626530e68f3d3b2c09fbb9654",
|
||||
"reference_path": "deepstream_app_tao_configs/nvinfer/config_infer_primary_dashcamnet.txt",
|
||||
"reference_config_sha256": "332b764b42238064a666bd5ec619c42905e44c099376247256996a03203ced93",
|
||||
"custom_mission_core_logic": false
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"library": "libnvds_nvmultiobjecttracker.so",
|
||||
"configuration": "config_tracker_NvDCF_perf.yml",
|
||||
"past_frame_output": false,
|
||||
"custom_association": false,
|
||||
"custom_hold_or_stitch": false
|
||||
},
|
||||
"output": {
|
||||
"frame_width": 800,
|
||||
"frame_height": 600,
|
||||
"overlay_codec": "H.264",
|
||||
"overlay_container": "MP4",
|
||||
"tracked_frames": "tracked-frames.jsonl"
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46g-rectified-detector-bakeoff-profile/v1",
|
||||
"profile_id": "e46g-k1-right-kb4-nvdewarper-ready-detector-bakeoff/v1",
|
||||
"comparison_contract": {
|
||||
"baseline_result_ids": [
|
||||
"e46e-ready-stack-d51fd744a86b0effa8685c7aa86d14dfd1b12e97bc8d68d0f53f467237b976bf",
|
||||
"e46f-dashcam-bakeoff-2b888a784ba06d4565d34a91fef58af1fc9090ed9dc298ad318010ff4da64507"
|
||||
],
|
||||
"controlled_change": "detector-provider-only",
|
||||
"held_constant": [
|
||||
"single recorded RIGHT source identity",
|
||||
"factory camera_1 KB4 calibration",
|
||||
"NVIDIA Gst-nvdewarper left/front/right projections",
|
||||
"source frame selection 1000..1599",
|
||||
"DeepStream 9.1 container digest",
|
||||
"FP16 precision",
|
||||
"NVIDIA NvDCF performance configuration",
|
||||
"960x544 rectified output plane"
|
||||
]
|
||||
},
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-camera-602ac89026ed12978619801d",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"segment_count": 4489,
|
||||
"stream_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"archive_index_sha256": "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14",
|
||||
"archive_summary_sha256": "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181"
|
||||
},
|
||||
"selection": {
|
||||
"first_source_frame_index": 1000,
|
||||
"last_source_frame_index": 1599,
|
||||
"frame_count": 600,
|
||||
"purpose": "contiguous model-independent urban gate containing dense vehicles, pedestrians and fisheye edge geometry"
|
||||
},
|
||||
"calibration": {
|
||||
"slot": "camera_1",
|
||||
"model": "KB4",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
"native_resolution": [
|
||||
4000,
|
||||
3000
|
||||
],
|
||||
"admitted_resolution": [
|
||||
800,
|
||||
600
|
||||
],
|
||||
"resize_contract": "linear 5x downscale without crop or warp",
|
||||
"intrinsic_fx_fy_cx_cy": [
|
||||
194.59817287616025,
|
||||
194.57531427932872,
|
||||
396.31861150187996,
|
||||
301.49644357408005
|
||||
],
|
||||
"distortion_kb4": [
|
||||
-0.023164451386679667,
|
||||
-0.0014974198594105452,
|
||||
-0.001039213149441563,
|
||||
-0.000035237331915978814
|
||||
]
|
||||
},
|
||||
"rectification": {
|
||||
"provider": "NVIDIA Gst-nvdewarper",
|
||||
"provider_version": "DeepStream 9.1",
|
||||
"projection": "fisheye-to-perspective",
|
||||
"projection_type": 4,
|
||||
"output_resolution": [
|
||||
960,
|
||||
544
|
||||
],
|
||||
"horizontal_fov_degrees": 100.0,
|
||||
"destination_focal_length": [
|
||||
402.76782296509447,
|
||||
402.76782296509447
|
||||
],
|
||||
"destination_principal_point": [
|
||||
479.5,
|
||||
271.5
|
||||
],
|
||||
"cuda_address_mode": "border",
|
||||
"expected_full_frame_count": 4488,
|
||||
"retained_source_frame_index_range": [
|
||||
0,
|
||||
4487
|
||||
],
|
||||
"excluded_source_tail_frame_count": 1,
|
||||
"tail_exclusion_reason": "DeepStream 9.1 nvv4l2decoder reproducibly omits only the terminal source frame at EOS; the admitted 1000..1599 gate is inside the timestamp-aligned retained prefix",
|
||||
"view_order": [
|
||||
"left",
|
||||
"front",
|
||||
"right"
|
||||
],
|
||||
"views": {
|
||||
"left": {
|
||||
"yaw_degrees": -90.0,
|
||||
"pitch_degrees": 0.0,
|
||||
"config_file": "e46g_nvdewarper_left.txt",
|
||||
"config_sha256": "257438d20a00ac495022b06a3a27b41822ccbb59bb1e9405501a2467b8456718"
|
||||
},
|
||||
"front": {
|
||||
"yaw_degrees": 0.0,
|
||||
"pitch_degrees": 0.0,
|
||||
"config_file": "e46g_nvdewarper_front.txt",
|
||||
"config_sha256": "f861e31278550bbe3fc82f41df4381c8a7aaf113a98a10761d98fa085c6a56b4"
|
||||
},
|
||||
"right": {
|
||||
"yaw_degrees": 90.0,
|
||||
"pitch_degrees": 0.0,
|
||||
"config_file": "e46g_nvdewarper_right.txt",
|
||||
"config_sha256": "16c9b6c22c8c9a0cf7a3e6127c7f5878d7890200dcc284d39f3ed0101277bb73"
|
||||
}
|
||||
},
|
||||
"official_documentation": "https://docs.nvidia.com/metropolis/deepstream/9.0/text/DS_plugin_gst-nvdewarper.html"
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:10eca409b3894e91c1bac915c9f1346307e56695e552487cbe8cf2f58a3f998f",
|
||||
"container_platform": "linux/amd64",
|
||||
"deepstream_version": "9.1",
|
||||
"network_during_inference": "none"
|
||||
},
|
||||
"candidates": {
|
||||
"trafficcamnet": {
|
||||
"name": "NVIDIA TrafficCamNet Transformer Lite",
|
||||
"architecture": "RT-DETR ResNet50",
|
||||
"version": "deployable_resnet50_v2.0",
|
||||
"precision": "FP16",
|
||||
"model_file": "resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"model_sha256": "545a447b913d54eee476381436ebea4ad2aa876cfbe0a6d9f3b0302f08a7415d",
|
||||
"model_url": "https://api.ngc.nvidia.com/v2/models/nvidia/tao/trafficcamnet_transformer_lite/versions/deployable_resnet50_v2.0/files/resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"deepstream_app_config": "e46g_trafficcamnet_deepstream_app.txt",
|
||||
"deepstream_app_config_sha256": "1a0ade326a80f9dcdf0b538948eb508972afb478466cf4294116b7361b62aadf",
|
||||
"detector_config": "e46e_trafficcamnet_rtdetr.txt",
|
||||
"detector_config_sha256": "be2c275c0a1cbef3ec6a5f1eef2ece50b7f3408fc1c56839c0b9863f6d577c76",
|
||||
"labels": [
|
||||
"background",
|
||||
"bicycle",
|
||||
"car",
|
||||
"person",
|
||||
"road_sign"
|
||||
],
|
||||
"custom_postprocessing": false
|
||||
},
|
||||
"dashcamnet": {
|
||||
"name": "NVIDIA DashCamNet",
|
||||
"architecture": "DetectNet_v2 ResNet18",
|
||||
"version": "pruned_onnx_v1.0.4",
|
||||
"precision": "FP16",
|
||||
"model_file": "resnet18_dashcamnet_pruned.onnx",
|
||||
"model_sha256": "d92f97bd840b68850a33c83e480fc6bd83b3097ff4cb517e0ea12671046bab7d",
|
||||
"model_url": "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_onnx_v1.0.4/files/resnet18_dashcamnet_pruned.onnx",
|
||||
"deepstream_app_config": "e46g_dashcamnet_deepstream_app.txt",
|
||||
"deepstream_app_config_sha256": "e25c4fb5d44dc964a28afd9b4c4449624e9cbe89e71b2734aae2a9b55765d603",
|
||||
"detector_config": "e46f_dashcamnet_detectnet.txt",
|
||||
"detector_config_sha256": "3493f5ba7236a949354133e437844b059f302a01c5f93b836f4f16f58a776238",
|
||||
"labels": [
|
||||
"car",
|
||||
"bicycle",
|
||||
"person",
|
||||
"road_sign"
|
||||
],
|
||||
"custom_postprocessing": false
|
||||
}
|
||||
},
|
||||
"trafficcamnet_parser": {
|
||||
"name": "NVIDIA DeepStream TAO custom bounding-box parser",
|
||||
"commit": "581889df47d6181110c758c10b872ca833a835e3",
|
||||
"symbol": "NvDsInferParseCustomDDETRTAO",
|
||||
"library_file": "libnvds_infercustomparser_tao.so",
|
||||
"library_sha256": "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee",
|
||||
"custom_mission_core_logic": false
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"library": "libnvds_nvmultiobjecttracker.so",
|
||||
"configuration": "config_tracker_NvDCF_perf.yml",
|
||||
"identity_scope": "view-local",
|
||||
"past_frame_output": false,
|
||||
"custom_association": false,
|
||||
"custom_hold_or_stitch": false
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46h-full-rectified-front-replay-profile/v1",
|
||||
"profile_id": "e46h-right-kb4-front-trafficcamnet-full-replay/v1",
|
||||
"baseline_result_id": "e46g-rectified-detector-bakeoff-9c4eb44cbb61199db0967bd9048712a0964e2dbf587651c9e0c6d808967675c6",
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-camera-602ac89026ed12978619801d",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"segment_count": 4489,
|
||||
"stream_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"archive_index_sha256": "e029815a60ad9fbfedb6169142c7449df2b119a51d1ce001f08806e04eb0be14",
|
||||
"archive_summary_sha256": "b280f40b198aad5d5335819107fb1405ad65d5695d187c61a2c027ad853a3181"
|
||||
},
|
||||
"selection": {
|
||||
"first_source_frame_index": 0,
|
||||
"last_source_frame_index": 4487,
|
||||
"frame_count": 4488,
|
||||
"excluded_source_tail_frame_count": 1,
|
||||
"tail_exclusion_reason": "DeepStream 9.1 nvv4l2decoder reproducibly omits only the terminal source frame at EOS; E46H admits the timestamp-aligned retained prefix 0..4487"
|
||||
},
|
||||
"calibration": {
|
||||
"slot": "camera_1",
|
||||
"model": "KB4",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
"native_resolution": [4000, 3000],
|
||||
"admitted_resolution": [800, 600],
|
||||
"resize_contract": "linear 5x downscale without crop or warp",
|
||||
"intrinsic_fx_fy_cx_cy": [
|
||||
194.59817287616025,
|
||||
194.57531427932872,
|
||||
396.31861150187996,
|
||||
301.49644357408005
|
||||
],
|
||||
"distortion_kb4": [
|
||||
-0.023164451386679667,
|
||||
-0.0014974198594105452,
|
||||
-0.001039213149441563,
|
||||
-0.000035237331915978814
|
||||
]
|
||||
},
|
||||
"rectification": {
|
||||
"provider": "NVIDIA Gst-nvdewarper",
|
||||
"provider_version": "DeepStream 9.1",
|
||||
"projection": "fisheye-to-perspective",
|
||||
"projection_type": 4,
|
||||
"view": "front",
|
||||
"yaw_degrees": 0.0,
|
||||
"pitch_degrees": 0.0,
|
||||
"output_resolution": [960, 544],
|
||||
"horizontal_fov_degrees": 100.0,
|
||||
"destination_focal_length": [402.76782296509447, 402.76782296509447],
|
||||
"destination_principal_point": [479.5, 271.5],
|
||||
"cuda_address_mode": "border",
|
||||
"config_file": "e46g_nvdewarper_front.txt",
|
||||
"config_sha256": "f861e31278550bbe3fc82f41df4381c8a7aaf113a98a10761d98fa085c6a56b4",
|
||||
"official_documentation": "https://docs.nvidia.com/metropolis/deepstream/9.0/text/DS_plugin_gst-nvdewarper.html"
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:10eca409b3894e91c1bac915c9f1346307e56695e552487cbe8cf2f58a3f998f",
|
||||
"container_platform": "linux/amd64",
|
||||
"deepstream_version": "9.1",
|
||||
"network_during_inference": "none"
|
||||
},
|
||||
"detector": {
|
||||
"name": "NVIDIA TrafficCamNet Transformer Lite",
|
||||
"architecture": "RT-DETR ResNet50",
|
||||
"version": "deployable_resnet50_v2.0",
|
||||
"precision": "FP16",
|
||||
"model_file": "resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"model_sha256": "545a447b913d54eee476381436ebea4ad2aa876cfbe0a6d9f3b0302f08a7415d",
|
||||
"model_url": "https://api.ngc.nvidia.com/v2/models/nvidia/tao/trafficcamnet_transformer_lite/versions/deployable_resnet50_v2.0/files/resnet50_trafficcamnet_rtdetr.fp16.onnx",
|
||||
"deepstream_app_config": "e46g_trafficcamnet_deepstream_app.txt",
|
||||
"deepstream_app_config_sha256": "1a0ade326a80f9dcdf0b538948eb508972afb478466cf4294116b7361b62aadf",
|
||||
"detector_config": "e46e_trafficcamnet_rtdetr.txt",
|
||||
"detector_config_sha256": "be2c275c0a1cbef3ec6a5f1eef2ece50b7f3408fc1c56839c0b9863f6d577c76",
|
||||
"labels": ["background", "bicycle", "car", "person", "road_sign"],
|
||||
"custom_postprocessing": false
|
||||
},
|
||||
"parser": {
|
||||
"name": "NVIDIA DeepStream TAO custom bounding-box parser",
|
||||
"repository": "https://github.com/NVIDIA/DeepStream.git",
|
||||
"commit": "581889df47d6181110c758c10b872ca833a835e3",
|
||||
"symbol": "NvDsInferParseCustomDDETRTAO",
|
||||
"library_file": "libnvds_infercustomparser_tao.so",
|
||||
"library_sha256": "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee",
|
||||
"custom_mission_core_logic": false
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"library": "libnvds_nvmultiobjecttracker.so",
|
||||
"configuration": "config_tracker_NvDCF_perf.yml",
|
||||
"identity_scope": "front-view route-local",
|
||||
"past_frame_output": false,
|
||||
"custom_association": false,
|
||||
"custom_hold_or_stitch": false
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46i-grounding-dino-full-replay-profile/v1",
|
||||
"profile_id": "e46i-right-kb4-front-grounding-dino-swin-tiny-full-replay/v1",
|
||||
"baseline_result_id": "e46h-full-rectified-front-replay-43f9d97c06387ffa3b7aa656c48c3cb475be5c40de1211853ae8cb3f5ff2e28f",
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"view": "front",
|
||||
"projection": "NVIDIA Gst-nvdewarper KB4 fisheye-to-perspective",
|
||||
"projection_resolution": [960, 544],
|
||||
"projection_fov_degrees": 100.0,
|
||||
"video_frame_count": 4488,
|
||||
"video_frame_rate": 10.0,
|
||||
"video_duration_seconds": 448.8,
|
||||
"video_sha256": "29bb32df8465ef53759bdf53b869f2edf2506fb70fd2a3fdf9dcdde02bf618e8"
|
||||
},
|
||||
"provider": {
|
||||
"name": "NVIDIA TAO Grounding DINO Swin-Tiny Commercial",
|
||||
"version": "1.0",
|
||||
"deployment_toolkit": "NVIDIA TAO Toolkit Deploy 7.0.1",
|
||||
"model_file": "grounding_dino_swin_tiny_commercial_deployable.onnx",
|
||||
"model_sha256": "6895acdc6b588e923f753e37b3bd18869e064256e5ecc1b2b9853e8c51125f94",
|
||||
"engine_file": "grounding_dino_swin_tiny_commercial_fp16.engine",
|
||||
"engine_sha256": "63213f60730ce8577047ff508dfe06113269347501e2dd832aeb56c89bc11489",
|
||||
"engine_precision": "FP16",
|
||||
"container_reference": "nvcr.io/nvidia/tao/tao-toolkit:7.0.1-deploy",
|
||||
"registry_manifest_sha256": "90d5d645ab75838db9ec7058855e5c283b59116ef0f1663a26b00b7c2c37280d",
|
||||
"registry_config_sha256": "ea2dda4441128170ae6eb798143517979ebaa523bab6d9c0d73547fdbae2b86a",
|
||||
"local_image_id_sha256": "2a3095330dd83e4314aada21fc7d184fcdb442a9131d591e5593ca5497394e7a",
|
||||
"custom_detector_or_postprocessing": false
|
||||
},
|
||||
"inference": {
|
||||
"spec_file": "e46i_grounding_dino_full_replay_spec.yaml",
|
||||
"spec_sha256": "c8513427f8bb8593a581bb29f286d89896e9639f98f851f2198aff86b8b541cb",
|
||||
"captions": ["car", "person", "bicycle", "road sign"],
|
||||
"confidence_threshold": 0.5,
|
||||
"batch_size": 1,
|
||||
"input_resolution": [960, 544],
|
||||
"gpu": "NVIDIA GeForce RTX 4090",
|
||||
"processed_frame_count": 4488,
|
||||
"elapsed_seconds": 748.0,
|
||||
"mean_frames_per_second": 6.0,
|
||||
"network_observation": "TAO downloaded the bert-base-uncased tokenizer from Hugging Face at run start; this execution is valid but not yet an offline-reproducible deployment"
|
||||
},
|
||||
"visual_shadow_gate": {
|
||||
"status": "completed",
|
||||
"threshold_changed_after_review": false,
|
||||
"legacy_large_false_background_cases": 5,
|
||||
"legacy_large_false_background_cases_suppressed": 5,
|
||||
"empty_anchor_cases": 1,
|
||||
"empty_anchor_cases_kept_empty": 1,
|
||||
"positive_anchor_cases": 5,
|
||||
"positive_anchor_cases_with_relevant_detection": 5,
|
||||
"known_semantic_limitations": [
|
||||
"one partially cropped person was missed in the stroller frame",
|
||||
"the stroller was labelled bicycle because stroller was not included in the fixed caption set"
|
||||
]
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"independent_truth": false,
|
||||
"candidate_accepted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
model_name: nvidia-grounding-dino-swin-tiny-commercial-v1.0
|
||||
results_dir: /workspace/e46i-full-results
|
||||
wandb:
|
||||
enable: false
|
||||
|
||||
model:
|
||||
backbone: swin_tiny_224_1k
|
||||
num_feature_levels: 4
|
||||
dec_layers: 6
|
||||
enc_layers: 6
|
||||
num_queries: 900
|
||||
dim_feedforward: 2048
|
||||
aux_loss: false
|
||||
log_scale: auto
|
||||
class_embed_bias: true
|
||||
|
||||
dataset:
|
||||
infer_data_sources:
|
||||
image_dir:
|
||||
- /workspace/full-input-images
|
||||
captions:
|
||||
- car
|
||||
- person
|
||||
- bicycle
|
||||
- road sign
|
||||
batch_size: 1
|
||||
workers: 1
|
||||
pin_memory: true
|
||||
max_labels: 80
|
||||
|
||||
inference:
|
||||
trt_engine: /workspace/models/grounding_dino_swin_tiny_commercial_fp16.engine
|
||||
batch_size: -1
|
||||
input_width: 960
|
||||
input_height: 544
|
||||
conf_threshold: 0.5
|
||||
outline_width: 3
|
||||
color_map:
|
||||
car: blue
|
||||
person: red
|
||||
bicycle: green
|
||||
road sign: yellow
|
||||
|
||||
gen_trt_engine:
|
||||
onnx_file: /workspace/models/grounding_dino_swin_tiny_commercial_deployable.onnx
|
||||
trt_engine: /workspace/models/grounding_dino_swin_tiny_commercial_fp16.engine
|
||||
batch_size: -1
|
||||
verbose: false
|
||||
tensorrt:
|
||||
workspace_size: 8192
|
||||
min_batch_size: 1
|
||||
opt_batch_size: 1
|
||||
max_batch_size: 1
|
||||
data_type: FP16
|
||||
@@ -0,0 +1,54 @@
|
||||
model_name: nvidia-grounding-dino-swin-tiny-commercial-v1.0
|
||||
results_dir: /workspace/e46i-results
|
||||
wandb:
|
||||
enable: false
|
||||
|
||||
model:
|
||||
backbone: swin_tiny_224_1k
|
||||
num_feature_levels: 4
|
||||
dec_layers: 6
|
||||
enc_layers: 6
|
||||
num_queries: 900
|
||||
dim_feedforward: 2048
|
||||
aux_loss: false
|
||||
log_scale: auto
|
||||
class_embed_bias: true
|
||||
|
||||
dataset:
|
||||
infer_data_sources:
|
||||
image_dir:
|
||||
- /workspace/input-images
|
||||
captions:
|
||||
- car
|
||||
- person
|
||||
- bicycle
|
||||
- road sign
|
||||
batch_size: 1
|
||||
workers: 1
|
||||
pin_memory: true
|
||||
max_labels: 80
|
||||
|
||||
inference:
|
||||
trt_engine: /workspace/models/grounding_dino_swin_tiny_commercial_fp16.engine
|
||||
batch_size: -1
|
||||
input_width: 960
|
||||
input_height: 544
|
||||
conf_threshold: 0.5
|
||||
outline_width: 3
|
||||
color_map:
|
||||
car: blue
|
||||
person: red
|
||||
bicycle: green
|
||||
road sign: yellow
|
||||
|
||||
gen_trt_engine:
|
||||
onnx_file: /workspace/models/grounding_dino_swin_tiny_commercial_deployable.onnx
|
||||
trt_engine: /workspace/models/grounding_dino_swin_tiny_commercial_fp16.engine
|
||||
batch_size: -1
|
||||
verbose: false
|
||||
tensorrt:
|
||||
workspace_size: 8192
|
||||
min_batch_size: 1
|
||||
opt_batch_size: 1
|
||||
max_batch_size: 1
|
||||
data_type: FP16
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"schema_version": "missioncore.e46j-raw-fisheye-realtime-profile/v1",
|
||||
"profile_id": "e46j-k1-right-raw-kb4-yolox-s-one-pass/v1",
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"job_id": "recorded-camera-602ac89026ed12978619801d",
|
||||
"stream_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"frame_count": 4489,
|
||||
"frame_rate": 10.003944527024467,
|
||||
"average_rate": "4489000/448723",
|
||||
"resolution": [800, 600],
|
||||
"calibration_slot": "camera_1",
|
||||
"calibration_model": "KB4",
|
||||
"calibration_sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9",
|
||||
"preprocessing_contract": "full admitted raw raster; valid-FOV fill and top-left letterbox only; no crop, dewarp or virtual view"
|
||||
},
|
||||
"valid_fov": {
|
||||
"result_id": "valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2",
|
||||
"mask_file": "mask.png"
|
||||
},
|
||||
"detector": {
|
||||
"id": "yolox_s",
|
||||
"version": 1,
|
||||
"architecture": "YOLOX-S",
|
||||
"source": "Megvii-BaseDetection/YOLOX release 0.1.1rc0",
|
||||
"license": "Apache-2.0",
|
||||
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063",
|
||||
"config_sha256": "5795c737a7935a655961b069e8404d336d891f9762fb6dffb93956a076479604",
|
||||
"runtime": "NVIDIA Triton 2.70.0 ONNX Runtime GPU backend",
|
||||
"input_name": "images",
|
||||
"output_name": "output",
|
||||
"input_shape": [1, 3, 640, 640],
|
||||
"classes": "COCO-80",
|
||||
"single_inference_per_source_frame": true
|
||||
},
|
||||
"preprocessing": {
|
||||
"color_order": "BGR",
|
||||
"resize": "bilinear-letterbox-top-left",
|
||||
"pad_value": 114,
|
||||
"valid_fov_fill_value": 114
|
||||
},
|
||||
"detection": {
|
||||
"minimum_score": 0.5,
|
||||
"nms_iou_threshold": 0.45,
|
||||
"target_class_ids": [0, 1, 2, 3, 5, 7],
|
||||
"minimum_box_area_pixels": 64.0,
|
||||
"maximum_box_area_fraction": 0.5,
|
||||
"minimum_valid_fov_fraction": 0.5,
|
||||
"require_center_inside_valid_fov": true,
|
||||
"custom_detector_logic": false,
|
||||
"route_specific_filtering": false
|
||||
},
|
||||
"acceptance": {
|
||||
"required_frame_count": 4489,
|
||||
"required_source_frame_rate": 10.003944527024467,
|
||||
"minimum_core_capacity_fps": 10.0,
|
||||
"maximum_core_path_p95_ms": 80.0,
|
||||
"maximum_inference_request_p95_ms": 60.0,
|
||||
"require_zero_failed_frames": true,
|
||||
"require_full_raw_fov": true
|
||||
},
|
||||
"authority": {
|
||||
"ground_truth": false,
|
||||
"provider_promoted": false,
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
},
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"candidate": {
|
||||
"architecture": "YOLOX-S",
|
||||
"candidate_id": "yolox-s-kb4-core3",
|
||||
"minimum_score": 0.25,
|
||||
"model_sha256": "c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063"
|
||||
},
|
||||
"evaluation": {
|
||||
"accuracy_gate": "blocked-until-independent-truth-seal",
|
||||
"metrics": [
|
||||
"AP50",
|
||||
"AP50:95",
|
||||
"per-class recall",
|
||||
"person/vehicle miss rate",
|
||||
"false-large-box rate",
|
||||
"valid-FOV leakage",
|
||||
"temporal class-count flicker"
|
||||
],
|
||||
"winner_policy": "explicit-review-only"
|
||||
},
|
||||
"mode": "recorded-replay-only",
|
||||
"pipeline_id": "kb4-core3-yolox-eomt-k1-lidar-e23-temporal/v1",
|
||||
"profile_id": "RAVNOVES00_RIGHT_YOLOX_TRUTH_ISLAND_V1",
|
||||
"schema_version": "missioncore.l34-right-yolox-truth-island-profile/v1",
|
||||
"source_session_id": "20260720T065719Z_viewer_live",
|
||||
"target_classes": [
|
||||
"person",
|
||||
"bicycle",
|
||||
"motorcycle",
|
||||
"car",
|
||||
"heavy_vehicle",
|
||||
"static_obstacle",
|
||||
"animal"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and validate the minimal content-addressed E46E Worker 006 package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.e46e_ready_stack import E46E_PACKAGE_SCHEMA, E46E_PROFILE_SCHEMA
|
||||
|
||||
_RUNTIME_FILES = {
|
||||
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||
"runtime/k1link/compute/__init__.py": None,
|
||||
"runtime/k1link/compute/e46e_ready_stack.py": "src/k1link/compute/e46e_ready_stack.py",
|
||||
"runtime/run_e46e_ready_stack.py": (
|
||||
"experiments/perception/worker/run_e46e_ready_stack.py"
|
||||
),
|
||||
"runtime/Invoke-E46EReadyStack.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46EReadyStack.ps1"
|
||||
),
|
||||
"runtime/Invoke-E46EReadyStackAsInteractiveUser.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46EReadyStackAsInteractiveUser.ps1"
|
||||
),
|
||||
"runtime/Build-E46ENvidiaTaoParser.ps1": (
|
||||
"experiments/perception/worker/Build-E46ENvidiaTaoParser.ps1"
|
||||
),
|
||||
"runtime/Invoke-E46ENvidiaTaoParserAsInteractiveUser.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46ENvidiaTaoParserAsInteractiveUser.ps1"
|
||||
),
|
||||
"runtime/e46e_deepstream_app.txt": (
|
||||
"experiments/perception/worker/e46e_deepstream_app.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_rtdetr.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_rtdetr.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_labels.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_labels.txt"
|
||||
),
|
||||
}
|
||||
_GENERATED_COMPUTE_INIT = (
|
||||
'"""Minimal E46E worker projection; import contract modules explicitly."""\n'
|
||||
)
|
||||
|
||||
|
||||
class E46EWorkerPackageError(RuntimeError):
|
||||
"""Raised when an E46E package is invalid."""
|
||||
|
||||
|
||||
def build_e46e_worker_package(
|
||||
*,
|
||||
repository_root: Path,
|
||||
profile_path: Path,
|
||||
parser_library_path: Path,
|
||||
output_root: Path,
|
||||
) -> Path:
|
||||
repository = repository_root.resolve(strict=True)
|
||||
profile_source = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_source)
|
||||
if profile.get("schema_version") != E46E_PROFILE_SCHEMA:
|
||||
raise E46EWorkerPackageError("E46E package profile is incompatible")
|
||||
parser_library = parser_library_path.resolve(strict=True)
|
||||
parser_profile = profile.get("parser")
|
||||
if (
|
||||
not isinstance(parser_profile, dict)
|
||||
or parser_profile.get("library_file") != parser_library.name
|
||||
or parser_profile.get("library_sha256") != _sha256(parser_library)
|
||||
or parser_library.is_symlink()
|
||||
):
|
||||
raise E46EWorkerPackageError("E46E NVIDIA parser library is incompatible")
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, relative in _RUNTIME_FILES.items():
|
||||
source = None if relative is None else repository / relative
|
||||
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||
raise E46EWorkerPackageError(f"E46E runtime source is invalid: {relative}")
|
||||
sources[target] = source
|
||||
sources["profile.json"] = profile_source
|
||||
sources[f"runtime/{parser_library.name}"] = parser_library
|
||||
descriptors = [
|
||||
_descriptor(relative, source)
|
||||
for relative, source in sorted(sources.items())
|
||||
]
|
||||
identity = {
|
||||
"schema_version": E46E_PACKAGE_SCHEMA,
|
||||
"classification": "minimal-stock-nvidia-recorded-right-worker-package",
|
||||
"profile_id": profile["profile_id"],
|
||||
"source_job_id": profile["source"]["job_id"],
|
||||
"profile_sha256": _sha256(profile_source),
|
||||
"artifact_paths": [row["path"] for row in descriptors],
|
||||
"source_artifacts": descriptors,
|
||||
"runtime": profile["runtime"],
|
||||
"detector": {
|
||||
"name": profile["detector"]["name"],
|
||||
"version": profile["detector"]["version"],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
},
|
||||
"parser": profile["parser"],
|
||||
"tracker": profile["tracker"],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
package_id = f"e46e-worker-package-{identity_sha256}"
|
||||
output = output_root.expanduser().absolute()
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = output / package_id
|
||||
if destination.exists():
|
||||
validate_e46e_worker_package(destination)
|
||||
return destination
|
||||
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
for relative, source in sources.items():
|
||||
target = staging / relative
|
||||
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if source is None:
|
||||
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||
else:
|
||||
shutil.copyfile(source, target)
|
||||
artifacts = [
|
||||
{
|
||||
"kind": relative,
|
||||
"path": relative,
|
||||
"byte_length": (staging / relative).stat().st_size,
|
||||
"sha256": _sha256(staging / relative),
|
||||
}
|
||||
for relative in sorted(sources)
|
||||
]
|
||||
_write_json(
|
||||
staging / "manifest.json",
|
||||
{
|
||||
"schema_version": E46E_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"artifacts": artifacts,
|
||||
},
|
||||
)
|
||||
validate_e46e_worker_package(staging, allow_staging=True)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
validate_e46e_worker_package(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def validate_e46e_worker_package(
|
||||
root: Path, *, allow_staging: bool = False
|
||||
) -> dict[str, Any]:
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
digest = (
|
||||
hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
if isinstance(identity, dict)
|
||||
else ""
|
||||
)
|
||||
package_id = f"e46e-worker-package-{digest}"
|
||||
valid_name = resolved.name == package_id or (
|
||||
allow_staging
|
||||
and resolved.name.startswith(f".{package_id}.")
|
||||
and resolved.name.endswith(".tmp")
|
||||
)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if (
|
||||
manifest.get("schema_version") != E46E_PACKAGE_SCHEMA
|
||||
or manifest.get("package_id") != package_id
|
||||
or manifest.get("identity_sha256") != digest
|
||||
or not valid_name
|
||||
or not isinstance(artifacts, list)
|
||||
):
|
||||
raise E46EWorkerPackageError("E46E worker package identity is invalid")
|
||||
expected_paths = set(identity.get("artifact_paths", []))
|
||||
actual_paths = {
|
||||
path.relative_to(resolved).as_posix()
|
||||
for path in resolved.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
if actual_paths != expected_paths | {"manifest.json"} or len(artifacts) != len(
|
||||
expected_paths
|
||||
):
|
||||
raise E46EWorkerPackageError("E46E worker package file set changed")
|
||||
observed: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E46EWorkerPackageError("E46E package artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_paths
|
||||
or relative in observed
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("kind") != relative
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E46EWorkerPackageError("E46E package artifact changed")
|
||||
observed.add(relative)
|
||||
if observed != expected_paths:
|
||||
raise E46EWorkerPackageError("E46E package coverage changed")
|
||||
return manifest
|
||||
|
||||
|
||||
def _descriptor(relative: str, source: Path | None) -> dict[str, Any]:
|
||||
payload = _GENERATED_COMPUTE_INIT.encode() if source is None else source.read_bytes()
|
||||
return {
|
||||
"path": relative,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E46EWorkerPackageError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--parser-library", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = build_e46e_worker_package(
|
||||
repository_root=args.repository_root,
|
||||
profile_path=args.profile,
|
||||
parser_library_path=args.parser_library,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(package)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and validate the minimal content-addressed E46F Worker 006 package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.e46f_dashcam_bakeoff import (
|
||||
E46F_PACKAGE_SCHEMA,
|
||||
E46F_PROFILE_SCHEMA,
|
||||
)
|
||||
|
||||
_RUNTIME_FILES = {
|
||||
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||
"runtime/k1link/compute/__init__.py": None,
|
||||
"runtime/k1link/compute/e46e_ready_stack.py": ("src/k1link/compute/e46e_ready_stack.py"),
|
||||
"runtime/k1link/compute/e46f_dashcam_bakeoff.py": (
|
||||
"src/k1link/compute/e46f_dashcam_bakeoff.py"
|
||||
),
|
||||
"runtime/run_e46f_dashcam_bakeoff.py": (
|
||||
"experiments/perception/worker/run_e46f_dashcam_bakeoff.py"
|
||||
),
|
||||
"runtime/Invoke-E46FDashCamBakeoff.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46FDashCamBakeoff.ps1"
|
||||
),
|
||||
"runtime/Invoke-E46FDashCamBakeoffAsInteractiveUser.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46FDashCamBakeoffAsInteractiveUser.ps1"
|
||||
),
|
||||
"runtime/e46f_deepstream_app.txt": ("experiments/perception/worker/e46f_deepstream_app.txt"),
|
||||
"runtime/e46f_dashcamnet_detectnet.txt": (
|
||||
"experiments/perception/worker/e46f_dashcamnet_detectnet.txt"
|
||||
),
|
||||
"runtime/e46f_dashcamnet_labels.txt": (
|
||||
"experiments/perception/worker/e46f_dashcamnet_labels.txt"
|
||||
),
|
||||
}
|
||||
_GENERATED_COMPUTE_INIT = (
|
||||
'"""Minimal E46F worker projection; import contract modules explicitly."""\n'
|
||||
)
|
||||
|
||||
|
||||
class E46FWorkerPackageError(RuntimeError):
|
||||
"""Raised when an E46F package is invalid."""
|
||||
|
||||
|
||||
def build_e46f_worker_package(
|
||||
*, repository_root: Path, profile_path: Path, output_root: Path
|
||||
) -> Path:
|
||||
repository = repository_root.resolve(strict=True)
|
||||
profile_source = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_source)
|
||||
if profile.get("schema_version") != E46F_PROFILE_SCHEMA:
|
||||
raise E46FWorkerPackageError("E46F package profile is incompatible")
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, relative in _RUNTIME_FILES.items():
|
||||
source = None if relative is None else repository / relative
|
||||
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||
raise E46FWorkerPackageError(f"E46F runtime source is invalid: {relative}")
|
||||
sources[target] = source
|
||||
sources["profile.json"] = profile_source
|
||||
descriptors = [_descriptor(relative, source) for relative, source in sorted(sources.items())]
|
||||
identity = {
|
||||
"schema_version": E46F_PACKAGE_SCHEMA,
|
||||
"classification": "minimal-stock-nvidia-detector-only-bakeoff-package",
|
||||
"profile_id": profile["profile_id"],
|
||||
"source_job_id": profile["source"]["job_id"],
|
||||
"baseline_result_id": profile["comparison_contract"]["baseline_result_id"],
|
||||
"profile_sha256": _sha256(profile_source),
|
||||
"artifact_paths": [row["path"] for row in descriptors],
|
||||
"source_artifacts": descriptors,
|
||||
"runtime": profile["runtime"],
|
||||
"detector": {
|
||||
"name": profile["detector"]["name"],
|
||||
"version": profile["detector"]["version"],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
},
|
||||
"postprocessor": profile["postprocessor"],
|
||||
"tracker": profile["tracker"],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
package_id = f"e46f-worker-package-{identity_sha256}"
|
||||
output = output_root.expanduser().absolute()
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = output / package_id
|
||||
if destination.exists():
|
||||
validate_e46f_worker_package(destination)
|
||||
return destination
|
||||
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
for relative, source in sources.items():
|
||||
target = staging / relative
|
||||
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if source is None:
|
||||
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||
else:
|
||||
shutil.copyfile(source, target)
|
||||
artifacts = [
|
||||
{
|
||||
"kind": relative,
|
||||
"path": relative,
|
||||
"byte_length": (staging / relative).stat().st_size,
|
||||
"sha256": _sha256(staging / relative),
|
||||
}
|
||||
for relative in sorted(sources)
|
||||
]
|
||||
_write_json(
|
||||
staging / "manifest.json",
|
||||
{
|
||||
"schema_version": E46F_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"artifacts": artifacts,
|
||||
},
|
||||
)
|
||||
validate_e46f_worker_package(staging, allow_staging=True)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
validate_e46f_worker_package(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def validate_e46f_worker_package(root: Path, *, allow_staging: bool = False) -> dict[str, Any]:
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
digest = (
|
||||
hashlib.sha256(_canonical_json(identity)).hexdigest() if isinstance(identity, dict) else ""
|
||||
)
|
||||
package_id = f"e46f-worker-package-{digest}"
|
||||
valid_name = resolved.name == package_id or (
|
||||
allow_staging
|
||||
and resolved.name.startswith(f".{package_id}.")
|
||||
and resolved.name.endswith(".tmp")
|
||||
)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if (
|
||||
manifest.get("schema_version") != E46F_PACKAGE_SCHEMA
|
||||
or manifest.get("package_id") != package_id
|
||||
or manifest.get("identity_sha256") != digest
|
||||
or not valid_name
|
||||
or not isinstance(artifacts, list)
|
||||
):
|
||||
raise E46FWorkerPackageError("E46F worker package identity is invalid")
|
||||
expected_paths = set(identity.get("artifact_paths", []))
|
||||
actual_paths = {
|
||||
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
|
||||
}
|
||||
if actual_paths != expected_paths | {"manifest.json"} or len(artifacts) != len(expected_paths):
|
||||
raise E46FWorkerPackageError("E46F worker package file set changed")
|
||||
observed: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E46FWorkerPackageError("E46F package artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_paths
|
||||
or relative in observed
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("kind") != relative
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E46FWorkerPackageError("E46F package artifact changed")
|
||||
observed.add(relative)
|
||||
if observed != expected_paths:
|
||||
raise E46FWorkerPackageError("E46F package coverage changed")
|
||||
return manifest
|
||||
|
||||
|
||||
def _descriptor(relative: str, source: Path | None) -> dict[str, Any]:
|
||||
payload = _GENERATED_COMPUTE_INIT.encode() if source is None else source.read_bytes()
|
||||
return {
|
||||
"path": relative,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E46FWorkerPackageError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = build_e46f_worker_package(
|
||||
repository_root=args.repository_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(package)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and validate the minimal content-addressed E46G Worker package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.e46g_rectified_detector_bakeoff import (
|
||||
E46G_PACKAGE_SCHEMA,
|
||||
E46G_PROFILE_SCHEMA,
|
||||
)
|
||||
|
||||
_RUNTIME_FILES = {
|
||||
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||
"runtime/k1link/compute/__init__.py": None,
|
||||
"runtime/k1link/compute/e46e_ready_stack.py": "src/k1link/compute/e46e_ready_stack.py",
|
||||
"runtime/k1link/compute/e46g_rectified_detector_bakeoff.py": (
|
||||
"src/k1link/compute/e46g_rectified_detector_bakeoff.py"
|
||||
),
|
||||
"runtime/run_e46g_rectified_detector_bakeoff.py": (
|
||||
"experiments/perception/worker/run_e46g_rectified_detector_bakeoff.py"
|
||||
),
|
||||
"runtime/Invoke-E46GRectifiedDetectorBakeoff.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46GRectifiedDetectorBakeoff.ps1"
|
||||
),
|
||||
"runtime/Invoke-E46GRectifiedDetectorBakeoffAsInteractiveUser.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46GRectifiedDetectorBakeoffAsInteractiveUser.ps1"
|
||||
),
|
||||
"runtime/Prepare-RectifiedCameraReplay.ps1": (
|
||||
"experiments/perception/worker/Prepare-RectifiedCameraReplay.ps1"
|
||||
),
|
||||
"runtime/e46g_nvdewarper_left.txt": ("experiments/perception/worker/e46g_nvdewarper_left.txt"),
|
||||
"runtime/e46g_nvdewarper_front.txt": (
|
||||
"experiments/perception/worker/e46g_nvdewarper_front.txt"
|
||||
),
|
||||
"runtime/e46g_nvdewarper_right.txt": (
|
||||
"experiments/perception/worker/e46g_nvdewarper_right.txt"
|
||||
),
|
||||
"runtime/e46g_trafficcamnet_deepstream_app.txt": (
|
||||
"experiments/perception/worker/e46g_trafficcamnet_deepstream_app.txt"
|
||||
),
|
||||
"runtime/e46g_dashcamnet_deepstream_app.txt": (
|
||||
"experiments/perception/worker/e46g_dashcamnet_deepstream_app.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_rtdetr.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_rtdetr.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_labels.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_labels.txt"
|
||||
),
|
||||
"runtime/e46f_dashcamnet_detectnet.txt": (
|
||||
"experiments/perception/worker/e46f_dashcamnet_detectnet.txt"
|
||||
),
|
||||
"runtime/e46f_dashcamnet_labels.txt": (
|
||||
"experiments/perception/worker/e46f_dashcamnet_labels.txt"
|
||||
),
|
||||
}
|
||||
_GENERATED_COMPUTE_INIT = (
|
||||
'"""Minimal E46G worker projection; import contract modules explicitly."""\n'
|
||||
)
|
||||
|
||||
|
||||
class E46GWorkerPackageError(RuntimeError):
|
||||
"""Raised when an E46G package is invalid."""
|
||||
|
||||
|
||||
def build_e46g_worker_package(
|
||||
*,
|
||||
repository_root: Path,
|
||||
profile_path: Path,
|
||||
parser_library_path: Path,
|
||||
output_root: Path,
|
||||
) -> Path:
|
||||
repository = repository_root.resolve(strict=True)
|
||||
profile_source = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_source)
|
||||
if profile.get("schema_version") != E46G_PROFILE_SCHEMA:
|
||||
raise E46GWorkerPackageError("E46G package profile is incompatible")
|
||||
parser_library = parser_library_path.resolve(strict=True)
|
||||
parser_profile = profile.get("trafficcamnet_parser")
|
||||
if (
|
||||
not isinstance(parser_profile, dict)
|
||||
or parser_profile.get("library_file") != parser_library.name
|
||||
or parser_profile.get("library_sha256") != _sha256(parser_library)
|
||||
or parser_library.is_symlink()
|
||||
):
|
||||
raise E46GWorkerPackageError("E46G NVIDIA parser library is incompatible")
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, relative in _RUNTIME_FILES.items():
|
||||
source = None if relative is None else repository / relative
|
||||
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||
raise E46GWorkerPackageError(f"E46G runtime source is invalid: {relative}")
|
||||
sources[target] = source
|
||||
sources["profile.json"] = profile_source
|
||||
sources[f"runtime/{parser_library.name}"] = parser_library
|
||||
descriptors = [_descriptor(relative, source) for relative, source in sorted(sources.items())]
|
||||
identity = {
|
||||
"schema_version": E46G_PACKAGE_SCHEMA,
|
||||
"classification": "minimal-factory-kb4-stock-nvidia-detector-bakeoff-package",
|
||||
"profile_id": profile["profile_id"],
|
||||
"source_job_id": profile["source"]["job_id"],
|
||||
"profile_sha256": _sha256(profile_source),
|
||||
"artifact_paths": [row["path"] for row in descriptors],
|
||||
"source_artifacts": descriptors,
|
||||
"runtime": profile["runtime"],
|
||||
"calibration": profile["calibration"],
|
||||
"rectification": profile["rectification"],
|
||||
"selection": profile["selection"],
|
||||
"candidates": {
|
||||
name: {
|
||||
"name": value["name"],
|
||||
"version": value["version"],
|
||||
"model_sha256": value["model_sha256"],
|
||||
"custom_postprocessing": value["custom_postprocessing"],
|
||||
}
|
||||
for name, value in sorted(profile["candidates"].items())
|
||||
},
|
||||
"tracker": profile["tracker"],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
package_id = f"e46g-worker-package-{identity_sha256}"
|
||||
output = output_root.expanduser().absolute()
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = output / package_id
|
||||
if destination.exists():
|
||||
validate_e46g_worker_package(destination)
|
||||
return destination
|
||||
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
for relative, source in sources.items():
|
||||
target = staging / relative
|
||||
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if source is None:
|
||||
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||
else:
|
||||
shutil.copyfile(source, target)
|
||||
artifacts = [
|
||||
{
|
||||
"kind": relative,
|
||||
"path": relative,
|
||||
"byte_length": (staging / relative).stat().st_size,
|
||||
"sha256": _sha256(staging / relative),
|
||||
}
|
||||
for relative in sorted(sources)
|
||||
]
|
||||
_write_json(
|
||||
staging / "manifest.json",
|
||||
{
|
||||
"schema_version": E46G_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"artifacts": artifacts,
|
||||
},
|
||||
)
|
||||
validate_e46g_worker_package(staging, allow_staging=True)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
validate_e46g_worker_package(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def validate_e46g_worker_package(root: Path, *, allow_staging: bool = False) -> dict[str, Any]:
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
digest = (
|
||||
hashlib.sha256(_canonical_json(identity)).hexdigest() if isinstance(identity, dict) else ""
|
||||
)
|
||||
package_id = f"e46g-worker-package-{digest}"
|
||||
valid_name = resolved.name == package_id or (
|
||||
allow_staging
|
||||
and resolved.name.startswith(f".{package_id}.")
|
||||
and resolved.name.endswith(".tmp")
|
||||
)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if (
|
||||
manifest.get("schema_version") != E46G_PACKAGE_SCHEMA
|
||||
or manifest.get("package_id") != package_id
|
||||
or manifest.get("identity_sha256") != digest
|
||||
or not valid_name
|
||||
or not isinstance(artifacts, list)
|
||||
):
|
||||
raise E46GWorkerPackageError("E46G worker package identity is invalid")
|
||||
expected_paths = set(identity.get("artifact_paths", []))
|
||||
actual_paths = {
|
||||
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
|
||||
}
|
||||
if actual_paths != expected_paths | {"manifest.json"} or len(artifacts) != len(expected_paths):
|
||||
raise E46GWorkerPackageError("E46G worker package file set changed")
|
||||
observed: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E46GWorkerPackageError("E46G package artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_paths
|
||||
or relative in observed
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("kind") != relative
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E46GWorkerPackageError("E46G package artifact changed")
|
||||
observed.add(relative)
|
||||
if observed != expected_paths:
|
||||
raise E46GWorkerPackageError("E46G package coverage changed")
|
||||
return manifest
|
||||
|
||||
|
||||
def _descriptor(relative: str, source: Path | None) -> dict[str, Any]:
|
||||
payload = _GENERATED_COMPUTE_INIT.encode() if source is None else source.read_bytes()
|
||||
return {
|
||||
"path": relative,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E46GWorkerPackageError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--parser-library", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = build_e46g_worker_package(
|
||||
repository_root=args.repository_root,
|
||||
profile_path=args.profile,
|
||||
parser_library_path=args.parser_library,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(package)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and validate the minimal content-addressed E46H Worker package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.e46h_full_rectified_front_replay import (
|
||||
E46H_PACKAGE_SCHEMA,
|
||||
E46H_PROFILE_SCHEMA,
|
||||
)
|
||||
|
||||
_RUNTIME_FILES = {
|
||||
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||
"runtime/k1link/compute/__init__.py": None,
|
||||
"runtime/k1link/compute/e46e_ready_stack.py": "src/k1link/compute/e46e_ready_stack.py",
|
||||
"runtime/k1link/compute/e46g_rectified_detector_bakeoff.py": (
|
||||
"src/k1link/compute/e46g_rectified_detector_bakeoff.py"
|
||||
),
|
||||
"runtime/k1link/compute/e46h_full_rectified_front_replay.py": (
|
||||
"src/k1link/compute/e46h_full_rectified_front_replay.py"
|
||||
),
|
||||
"runtime/run_e46h_full_rectified_front_replay.py": (
|
||||
"experiments/perception/worker/run_e46h_full_rectified_front_replay.py"
|
||||
),
|
||||
"runtime/Invoke-E46HFullRectifiedFrontReplay.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46HFullRectifiedFrontReplay.ps1"
|
||||
),
|
||||
"runtime/Invoke-E46HFullRectifiedFrontReplayAsInteractiveUser.ps1": (
|
||||
"experiments/perception/worker/Invoke-E46HFullRectifiedFrontReplayAsInteractiveUser.ps1"
|
||||
),
|
||||
"runtime/Prepare-RectifiedCameraReplay.ps1": (
|
||||
"experiments/perception/worker/Prepare-RectifiedCameraReplay.ps1"
|
||||
),
|
||||
"runtime/e46g_nvdewarper_front.txt": (
|
||||
"experiments/perception/worker/e46g_nvdewarper_front.txt"
|
||||
),
|
||||
"runtime/e46g_trafficcamnet_deepstream_app.txt": (
|
||||
"experiments/perception/worker/e46g_trafficcamnet_deepstream_app.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_rtdetr.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_rtdetr.txt"
|
||||
),
|
||||
"runtime/e46e_trafficcamnet_labels.txt": (
|
||||
"experiments/perception/worker/e46e_trafficcamnet_labels.txt"
|
||||
),
|
||||
}
|
||||
_GENERATED_COMPUTE_INIT = (
|
||||
'"""Minimal E46H worker projection; import contract modules explicitly."""\n'
|
||||
)
|
||||
|
||||
|
||||
class E46HWorkerPackageError(RuntimeError):
|
||||
"""Raised when an E46H package is invalid."""
|
||||
|
||||
|
||||
def build_e46h_worker_package(
|
||||
*,
|
||||
repository_root: Path,
|
||||
profile_path: Path,
|
||||
parser_library_path: Path,
|
||||
output_root: Path,
|
||||
) -> Path:
|
||||
repository = repository_root.resolve(strict=True)
|
||||
profile_source = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_source)
|
||||
if profile.get("schema_version") != E46H_PROFILE_SCHEMA:
|
||||
raise E46HWorkerPackageError("E46H package profile is incompatible")
|
||||
parser_library = parser_library_path.resolve(strict=True)
|
||||
parser = profile.get("parser")
|
||||
if (
|
||||
not isinstance(parser, dict)
|
||||
or parser.get("library_file") != parser_library.name
|
||||
or parser.get("library_sha256") != _sha256(parser_library)
|
||||
or parser_library.is_symlink()
|
||||
):
|
||||
raise E46HWorkerPackageError("E46H NVIDIA parser library is incompatible")
|
||||
sources: dict[str, Path | None] = {}
|
||||
for target, relative in _RUNTIME_FILES.items():
|
||||
source = None if relative is None else repository / relative
|
||||
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||
raise E46HWorkerPackageError(f"E46H runtime source is invalid: {relative}")
|
||||
sources[target] = source
|
||||
sources["profile.json"] = profile_source
|
||||
sources[f"runtime/{parser_library.name}"] = parser_library
|
||||
descriptors = [_descriptor(relative, source) for relative, source in sorted(sources.items())]
|
||||
identity = {
|
||||
"schema_version": E46H_PACKAGE_SCHEMA,
|
||||
"classification": "minimal-full-front-stock-nvidia-replay-package",
|
||||
"profile_id": profile["profile_id"],
|
||||
"baseline_result_id": profile["baseline_result_id"],
|
||||
"source_job_id": profile["source"]["job_id"],
|
||||
"profile_sha256": _sha256(profile_source),
|
||||
"artifact_paths": [row["path"] for row in descriptors],
|
||||
"source_artifacts": descriptors,
|
||||
"runtime": profile["runtime"],
|
||||
"calibration": profile["calibration"],
|
||||
"rectification": profile["rectification"],
|
||||
"selection": profile["selection"],
|
||||
"detector": {
|
||||
"name": profile["detector"]["name"],
|
||||
"version": profile["detector"]["version"],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
"custom_postprocessing": profile["detector"]["custom_postprocessing"],
|
||||
},
|
||||
"tracker": profile["tracker"],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
package_id = f"e46h-worker-package-{identity_sha256}"
|
||||
output = output_root.expanduser().absolute()
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = output / package_id
|
||||
if destination.exists():
|
||||
validate_e46h_worker_package(destination)
|
||||
return destination
|
||||
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
for relative, source in sources.items():
|
||||
target = staging / relative
|
||||
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if source is None:
|
||||
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||
else:
|
||||
shutil.copyfile(source, target)
|
||||
artifacts = [
|
||||
{
|
||||
"kind": relative,
|
||||
"path": relative,
|
||||
"byte_length": (staging / relative).stat().st_size,
|
||||
"sha256": _sha256(staging / relative),
|
||||
}
|
||||
for relative in sorted(sources)
|
||||
]
|
||||
_write_json(
|
||||
staging / "manifest.json",
|
||||
{
|
||||
"schema_version": E46H_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"artifacts": artifacts,
|
||||
},
|
||||
)
|
||||
validate_e46h_worker_package(staging, allow_staging=True)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
validate_e46h_worker_package(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def validate_e46h_worker_package(root: Path, *, allow_staging: bool = False) -> dict[str, Any]:
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
digest = (
|
||||
hashlib.sha256(_canonical_json(identity)).hexdigest() if isinstance(identity, dict) else ""
|
||||
)
|
||||
package_id = f"e46h-worker-package-{digest}"
|
||||
valid_name = resolved.name == package_id or (
|
||||
allow_staging
|
||||
and resolved.name.startswith(f".{package_id}.")
|
||||
and resolved.name.endswith(".tmp")
|
||||
)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if (
|
||||
manifest.get("schema_version") != E46H_PACKAGE_SCHEMA
|
||||
or manifest.get("package_id") != package_id
|
||||
or manifest.get("identity_sha256") != digest
|
||||
or not valid_name
|
||||
or not isinstance(artifacts, list)
|
||||
):
|
||||
raise E46HWorkerPackageError("E46H worker package identity is invalid")
|
||||
expected_paths = set(identity.get("artifact_paths", []))
|
||||
actual_paths = {
|
||||
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
|
||||
}
|
||||
if actual_paths != expected_paths | {"manifest.json"} or len(artifacts) != len(expected_paths):
|
||||
raise E46HWorkerPackageError("E46H worker package file set changed")
|
||||
observed: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E46HWorkerPackageError("E46H package artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_paths
|
||||
or relative in observed
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("kind") != relative
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E46HWorkerPackageError("E46H package artifact changed")
|
||||
observed.add(relative)
|
||||
if observed != expected_paths:
|
||||
raise E46HWorkerPackageError("E46H package coverage changed")
|
||||
return manifest
|
||||
|
||||
|
||||
def _descriptor(relative: str, source: Path | None) -> dict[str, Any]:
|
||||
payload = _GENERATED_COMPUTE_INIT.encode() if source is None else source.read_bytes()
|
||||
return {
|
||||
"path": relative,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E46HWorkerPackageError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--parser-library", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = build_e46h_worker_package(
|
||||
repository_root=args.repository_root,
|
||||
profile_path=args.profile,
|
||||
parser_library_path=args.parser_library,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(package)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable E46B recorded RIGHT temporal-motion result."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46b_temporal_motion import build_e46b_temporal_motion
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
result = build_e46b_temporal_motion(
|
||||
e46a_root=(
|
||||
root
|
||||
/ ".runtime/compute-experiments/e46a/ai-engineering-preannotations"
|
||||
/ (
|
||||
"e46a-ai-engineering-preannotation-"
|
||||
"37cab05e1168cd6004202b890f2c7a877ab02d036afd0891a11a4df61d4d26bf"
|
||||
)
|
||||
),
|
||||
e26_root=(
|
||||
root
|
||||
/ ".runtime/compute-experiments/e10/worker-results"
|
||||
/ (
|
||||
"e10-integrated-perception-"
|
||||
"459aac93918d8f6414b342986ccc6968fefcef6c1f3a78a5254df0b565255ad2"
|
||||
)
|
||||
),
|
||||
output_root=root / ".runtime/compute-experiments/e46b/temporal-motion",
|
||||
)
|
||||
print(result["result_id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable E46C full recorded route/world-track result."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46c_full_replay_world_tracks import (
|
||||
build_e46c_full_replay_world_tracks,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
result = build_e46c_full_replay_world_tracks(
|
||||
e46a_root=(
|
||||
root
|
||||
/ ".runtime/compute-experiments/e46a/ai-engineering-preannotations"
|
||||
/ (
|
||||
"e46a-ai-engineering-preannotation-"
|
||||
"37cab05e1168cd6004202b890f2c7a877ab02d036afd0891a11a4df61d4d26bf"
|
||||
)
|
||||
),
|
||||
e26_root=(
|
||||
root
|
||||
/ ".runtime/compute-experiments/e10/worker-results"
|
||||
/ (
|
||||
"e10-integrated-perception-"
|
||||
"459aac93918d8f6414b342986ccc6968fefcef6c1f3a78a5254df0b565255ad2"
|
||||
)
|
||||
),
|
||||
output_root=(
|
||||
root / ".runtime/compute-experiments/e46c/full-replay-world-tracks"
|
||||
),
|
||||
)
|
||||
print(result["result_id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable E46D full-route temporal-failure audit."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46d_temporal_failure_audit import (
|
||||
build_e46d_temporal_failure_audit,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
result = build_e46d_temporal_failure_audit(
|
||||
e46c_root=(
|
||||
root
|
||||
/ ".runtime/compute-experiments/e46c/full-replay-world-tracks"
|
||||
/ (
|
||||
"e46c-full-replay-world-tracks-"
|
||||
"98ca4aeb9839082be64c1ce375ea773cff290e08cf085d8d24bb29e11d6fba8d"
|
||||
)
|
||||
),
|
||||
e26_results_root=root / ".runtime/compute-experiments/e10/worker-results",
|
||||
output_root=root / ".runtime/compute-experiments/e46d/temporal-failure-audits",
|
||||
)
|
||||
print(result["result_id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze validated E46I Grounding DINO full-route evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46i_grounding_dino_full_replay import (
|
||||
build_e46i_grounding_dino_full_replay,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46i_grounding_dino_full_replay(
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(result["result_root"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze one validated E46J worker run as an immutable LAB result."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46j_raw_fisheye_realtime import (
|
||||
build_e46j_raw_fisheye_realtime,
|
||||
)
|
||||
|
||||
|
||||
def arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = arguments()
|
||||
frozen = build_e46j_raw_fisheye_realtime(
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": frozen["result_id"],
|
||||
"result_root": str(frozen["result_root"]),
|
||||
"status": frozen["report"]["status"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze the first recorded right-camera benchmark candidate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34_right_yolox_truth_island_freeze import (
|
||||
build_l34_right_yolox_truth_island_freeze,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--truth-island-root", type=Path, required=True)
|
||||
parser.add_argument("--detector-qualification-root", type=Path, required=True)
|
||||
parser.add_argument("--l33-result-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34_right_yolox_truth_island_freeze(
|
||||
profile_path=args.profile,
|
||||
truth_island_root=args.truth_island_root,
|
||||
detector_qualification_root=args.detector_qualification_root,
|
||||
l33_result_root=args.l33_result_root,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"status": result.report["status"],
|
||||
"metrics": result.report["metrics"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the non-truth assisted error audit for the exact L3.4 freeze."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34a_assisted_yolox_error_audit import (
|
||||
build_l34a_assisted_yolox_error_audit,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--l34-freeze-root", type=Path, required=True)
|
||||
parser.add_argument("--annotation-session", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34a_assisted_yolox_error_audit(
|
||||
l34_freeze_root=args.l34_freeze_root,
|
||||
annotation_session_path=args.annotation_session,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
report = result["report"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": report["status"],
|
||||
"metrics": report["metrics"],
|
||||
"ground_truth": report["ground_truth"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the L3.4B nested-box consolidation shadow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34b_nested_box_consolidation_shadow import (
|
||||
build_l34b_nested_box_consolidation_shadow,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--l34-freeze-root", type=Path, required=True)
|
||||
parser.add_argument("--l34a-audit-root", type=Path, required=True)
|
||||
parser.add_argument("--annotation-session", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34b_nested_box_consolidation_shadow(
|
||||
l34_freeze_root=args.l34_freeze_root,
|
||||
l34a_audit_root=args.l34a_audit_root,
|
||||
annotation_session_path=args.annotation_session,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
report = result["report"]
|
||||
print(json.dumps({
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": report["status"],
|
||||
"metrics": report["metrics"],
|
||||
"ground_truth": report["ground_truth"],
|
||||
}, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable L3.4C temporal tile-seam stitch shadow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34c_tile_seam_stitch_shadow import (
|
||||
build_l34c_tile_seam_stitch_shadow,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--l34-freeze-root", type=Path, required=True)
|
||||
parser.add_argument("--l34a-audit-root", type=Path, required=True)
|
||||
parser.add_argument("--annotation-session", type=Path, required=True)
|
||||
parser.add_argument("--detector-qualification-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34c_tile_seam_stitch_shadow(
|
||||
l34_freeze_root=args.l34_freeze_root,
|
||||
l34a_audit_root=args.l34a_audit_root,
|
||||
annotation_session_path=args.annotation_session,
|
||||
detector_qualification_root=args.detector_qualification_root,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
report = result["report"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": report["status"],
|
||||
"metrics": report["metrics"],
|
||||
"ground_truth": report["ground_truth"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable L3.4D cumulative post-processing candidate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34d_cumulative_postprocessing_candidate import (
|
||||
build_l34d_cumulative_postprocessing_candidate,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--l34a-audit-root", type=Path, required=True)
|
||||
parser.add_argument("--l34b-shadow-root", type=Path, required=True)
|
||||
parser.add_argument("--l34c-shadow-root", type=Path, required=True)
|
||||
parser.add_argument("--annotation-session", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34d_cumulative_postprocessing_candidate(
|
||||
l34a_audit_root=args.l34a_audit_root,
|
||||
l34b_shadow_root=args.l34b_shadow_root,
|
||||
l34c_shadow_root=args.l34c_shadow_root,
|
||||
annotation_session_path=args.annotation_session,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
report = result["report"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": report["status"],
|
||||
"metrics": report["metrics"],
|
||||
"ground_truth": report["ground_truth"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the non-truth L3.4E manual self-review diagnostic."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l34e_self_review_diagnostic import (
|
||||
build_l34e_self_review_diagnostic,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--l34d-candidate-root", type=Path, required=True)
|
||||
parser.add_argument("--annotation-session", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l34e_self_review_diagnostic(
|
||||
l34d_candidate_root=args.l34d_candidate_root,
|
||||
annotation_session_path=args.annotation_session,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
report = result["report"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": report["status"],
|
||||
"metrics": report["metrics"],
|
||||
"decision": report["decision"],
|
||||
"ground_truth": report["ground_truth"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate exact L3.4 predictions after the independent E48 truth seal."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.l35_right_yolox_truth_evaluation import (
|
||||
build_l35_right_yolox_truth_evaluation,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--truth-island-root", type=Path, required=True)
|
||||
parser.add_argument("--truth-seal-root", type=Path, required=True)
|
||||
parser.add_argument("--l34-freeze-root", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_l35_right_yolox_truth_evaluation(
|
||||
truth_island_root=args.truth_island_root,
|
||||
truth_seal_root=args.truth_seal_root,
|
||||
l34_freeze_root=args.l34_freeze_root,
|
||||
valid_fov_root=args.valid_fov_root,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"status": result["report"]["status"],
|
||||
"candidates": result["report"]["candidates"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,509 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Attach canonical E29 LiDAR geometry to rectified YOLOX ByteTrack objects."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import resource
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _install_compute_import_boundary() -> None:
|
||||
"""Load pure compute modules without importing the application/Rerun graph.
|
||||
|
||||
The product packages still expose legacy eager ``__init__`` imports. A bounded
|
||||
worker replay must not acquire MQTT, Rerun or web dependencies merely to read
|
||||
immutable LiDAR arrays and run calibrated geometry. These namespace packages
|
||||
preserve normal submodule loading while deliberately skipping those facades.
|
||||
"""
|
||||
|
||||
import k1link
|
||||
import k1link.device_plugins
|
||||
|
||||
k1link_root = Path(next(iter(k1link.__path__))).resolve(strict=True)
|
||||
|
||||
def namespace(name: str, path: Path) -> types.ModuleType:
|
||||
module = types.ModuleType(name)
|
||||
module.__package__ = name
|
||||
module.__path__ = [str(path)] # type: ignore[attr-defined]
|
||||
sys.modules[name] = module
|
||||
parent_name, _, child_name = name.rpartition(".")
|
||||
if parent_name:
|
||||
setattr(sys.modules[parent_name], child_name, module)
|
||||
return module
|
||||
|
||||
namespace("k1link.compute", k1link_root / "compute")
|
||||
xgrids = namespace(
|
||||
"k1link.device_plugins.xgrids_k1",
|
||||
k1link_root / "device_plugins" / "xgrids_k1",
|
||||
)
|
||||
analyze = namespace(
|
||||
"k1link.device_plugins.xgrids_k1.analyze",
|
||||
k1link_root / "device_plugins" / "xgrids_k1" / "analyze",
|
||||
)
|
||||
|
||||
replay = types.ModuleType("k1link.compute.lidar_replay")
|
||||
replay.LIDAR_REPLAY_PACK_SCHEMA = "missioncore.lidar-replay-pack/v2"
|
||||
replay.LidarReplayPackV2 = type("LidarReplayPackV2", (), {})
|
||||
sys.modules[replay.__name__] = replay
|
||||
|
||||
projection = importlib.import_module(
|
||||
"k1link.device_plugins.xgrids_k1.analyze.calibrated_projection"
|
||||
)
|
||||
for export in (
|
||||
"CalibratedProjectionError",
|
||||
"Kb4ProjectionProfile",
|
||||
"ProjectedPointCloud",
|
||||
"map_points_to_lidar",
|
||||
"project_map_points_kb4",
|
||||
):
|
||||
setattr(analyze, export, getattr(projection, export))
|
||||
setattr(xgrids, "analyze", analyze)
|
||||
|
||||
|
||||
_install_compute_import_boundary()
|
||||
|
||||
from k1link.compute.lidar_field_review import E10LidarFieldSource
|
||||
from k1link.compute.lidar_local_surface import K1LocalSurfaceV1
|
||||
from k1link.compute.semantic_geometry_fusion import (
|
||||
DEFAULT_CAMERA_GEOMETRY_FUSION_PROFILE,
|
||||
evaluate_camera_geometry_frame,
|
||||
projection_profile_from_source,
|
||||
)
|
||||
from k1link.device_plugins.xgrids_k1.analyze.calibrated_projection import (
|
||||
project_map_points_kb4,
|
||||
)
|
||||
|
||||
|
||||
RESULT_SCHEMA = "missioncore.rectified-camera-lidar-fusion/v1"
|
||||
FRAME_SCHEMA = "missioncore.rectified-camera-lidar-frame/v1"
|
||||
AUTHORITY = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
LABEL_GROUP = {
|
||||
"person": "person",
|
||||
"bicycle": "two-wheel",
|
||||
"motorcycle": "two-wheel",
|
||||
"car": "vehicle",
|
||||
"bus": "vehicle",
|
||||
"truck": "vehicle",
|
||||
}
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--detector-result", type=Path, required=True)
|
||||
parser.add_argument("--source-pack", type=Path, required=True)
|
||||
parser.add_argument("--local-surface", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.resolve(strict=True).read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _distribution(values: list[float]) -> dict[str, float]:
|
||||
ordered = np.asarray(values, dtype=np.float64)
|
||||
return {
|
||||
"minimum": round(float(np.min(ordered)), 6),
|
||||
"mean": round(float(np.mean(ordered)), 6),
|
||||
"p50": round(float(np.percentile(ordered, 50)), 6),
|
||||
"p95": round(float(np.percentile(ordered, 95)), 6),
|
||||
"p99": round(float(np.percentile(ordered, 99)), 6),
|
||||
"maximum": round(float(np.max(ordered)), 6),
|
||||
}
|
||||
|
||||
|
||||
def _process_peak_rss_mib() -> float:
|
||||
value = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
divisor = 1024.0 * 1024.0 if sys.platform == "darwin" else 1024.0
|
||||
return value / divisor
|
||||
|
||||
|
||||
def _track_objects(frame: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
tracks = frame.get("tracks")
|
||||
if not isinstance(tracks, list):
|
||||
raise RuntimeError("Rectified detector frame has no track list")
|
||||
objects: list[dict[str, Any]] = []
|
||||
for track in tracks:
|
||||
if not isinstance(track, dict):
|
||||
raise RuntimeError("Rectified detector track is invalid")
|
||||
label = track.get("label")
|
||||
track_id = track.get("track_id")
|
||||
bbox = track.get("bbox_xyxy")
|
||||
score = track.get("score")
|
||||
if (
|
||||
label not in LABEL_GROUP
|
||||
or not isinstance(track_id, int)
|
||||
or isinstance(track_id, bool)
|
||||
or not isinstance(bbox, list)
|
||||
or len(bbox) != 4
|
||||
or any(
|
||||
not isinstance(value, int | float)
|
||||
or isinstance(value, bool)
|
||||
or not math.isfinite(float(value))
|
||||
for value in bbox
|
||||
)
|
||||
or not isinstance(score, int | float)
|
||||
or isinstance(score, bool)
|
||||
or not math.isfinite(float(score))
|
||||
):
|
||||
raise RuntimeError("Rectified detector track contract changed")
|
||||
objects.append(
|
||||
{
|
||||
"source_track_id": track_id,
|
||||
"track_id": track_id,
|
||||
"label": label,
|
||||
"association_group": LABEL_GROUP[label],
|
||||
"score": float(score),
|
||||
"bbox_xyxy": [float(value) for value in bbox],
|
||||
"cuboid_status": "camera-track-current-no-cuboid",
|
||||
"camera_motion_state": "bytetrack-current",
|
||||
"camera_motion_confidence": None,
|
||||
"motion_state": "camera-track-current",
|
||||
"motion_status": "metric-velocity-not-yet-published",
|
||||
}
|
||||
)
|
||||
return objects
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = _arguments()
|
||||
detector_root = arguments.detector_result.resolve(strict=True)
|
||||
if detector_root.is_symlink():
|
||||
raise RuntimeError("Detector result cannot be a symlink")
|
||||
qualification_path = detector_root / "qualification.json"
|
||||
detector_frames_path = detector_root / "frames.jsonl"
|
||||
qualification = _read_object(qualification_path)
|
||||
if (
|
||||
qualification.get("schema_version")
|
||||
!= "missioncore.rectified-yolox-qualification/v1"
|
||||
or qualification.get("state") != "accepted"
|
||||
or qualification.get("source", {}).get("frame_count") != 4489
|
||||
or qualification.get("source", {}).get("session_id")
|
||||
!= "20260720T065719Z_viewer_live"
|
||||
or qualification.get("pipeline", {}).get("id")
|
||||
!= "k1-kb4-core3-yolox-bytetrack/v1"
|
||||
or qualification.get("acceptance", {}).get("accepted") is not True
|
||||
):
|
||||
raise RuntimeError("Rectified detector qualification is not accepted")
|
||||
|
||||
source = E10LidarFieldSource(arguments.source_pack)
|
||||
surface = K1LocalSurfaceV1(arguments.local_surface)
|
||||
try:
|
||||
if (
|
||||
source.frame_count != 4489
|
||||
or source.identity.get("session_id")
|
||||
!= qualification["source"]["session_id"]
|
||||
or source.identity.get("source_id")
|
||||
!= qualification["source"]["source_id"]
|
||||
or surface.identity.get("source_pack_id") != source.pack_id
|
||||
or surface.identity.get("frame_count") != source.frame_count
|
||||
):
|
||||
raise RuntimeError("Detector, LiDAR and local surface are not source-aligned")
|
||||
identity = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"source": {
|
||||
"session_id": source.identity["session_id"],
|
||||
"source_pack_id": source.pack_id,
|
||||
"source_artifact_sha256": source.manifest["artifact"]["sha256"],
|
||||
"local_surface_model_id": surface.model_id,
|
||||
"local_surface_logical_sha256": surface.identity[
|
||||
"logical_content_sha256"
|
||||
],
|
||||
"detector_qualification_sha256": _sha256(qualification_path),
|
||||
"detector_frames_sha256": _sha256(detector_frames_path),
|
||||
},
|
||||
"pipeline": qualification["pipeline"],
|
||||
"association": DEFAULT_CAMERA_GEOMETRY_FUSION_PROFILE.to_dict(),
|
||||
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical(identity)).hexdigest()
|
||||
result_id = f"rectified-camera-lidar-fusion-{identity_sha256}"
|
||||
destination = arguments.output_root.expanduser().absolute() / result_id
|
||||
if destination.exists():
|
||||
print(json.dumps({"result_id": result_id, "result_root": str(destination)}))
|
||||
return
|
||||
destination.parent.mkdir(parents=True, mode=0o700, exist_ok=True)
|
||||
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700)
|
||||
started = time.perf_counter()
|
||||
rss_start = _process_peak_rss_mib()
|
||||
arrays = source.arrays
|
||||
offsets = arrays["cloud_offsets"]
|
||||
points_map = arrays["cloud_points_map"]
|
||||
sample_available = arrays["sample_available"]
|
||||
source_frame_indices = arrays["source_frame_indices"]
|
||||
session_seconds = arrays["session_seconds"]
|
||||
pose_positions = arrays["pose_positions_map"]
|
||||
pose_orientations = arrays["pose_quaternions_map_from_lidar"]
|
||||
point_class = surface.arrays["point_class"]
|
||||
point_height = surface.arrays["point_height_m"]
|
||||
surface_frame_valid = surface.arrays["frame_valid"]
|
||||
projection = projection_profile_from_source(source)
|
||||
latencies: list[float] = []
|
||||
status_counts: Counter[str] = Counter()
|
||||
ranged_by_label: Counter[str] = Counter()
|
||||
observations_by_label: Counter[str] = Counter()
|
||||
observation_count = 0
|
||||
ranged_count = 0
|
||||
range_estimate_count = 0
|
||||
unqualified_range_estimates_withheld = 0
|
||||
unqualified_metric_range_count = 0
|
||||
frames_with_tracks = 0
|
||||
frame_count = 0
|
||||
frames_path = staging / "camera-lidar-frames.jsonl"
|
||||
try:
|
||||
with (
|
||||
detector_frames_path.open("r", encoding="utf-8") as detector_stream,
|
||||
frames_path.open("x", encoding="utf-8", newline="\n") as output,
|
||||
):
|
||||
for line in detector_stream:
|
||||
frame_started = time.perf_counter()
|
||||
detector = json.loads(line)
|
||||
if (
|
||||
not isinstance(detector, dict)
|
||||
or detector.get("schema_version")
|
||||
!= "missioncore.rectified-yolox-frame/v1"
|
||||
or detector.get("frame_index") != frame_count
|
||||
):
|
||||
raise RuntimeError("Detector frame order or schema changed")
|
||||
objects = _track_objects(detector)
|
||||
frames_with_tracks += int(bool(objects))
|
||||
start = int(offsets[frame_count])
|
||||
end = int(offsets[frame_count + 1])
|
||||
points = np.asarray(points_map[start:end], dtype=np.float64)
|
||||
source_available = bool(sample_available[frame_count])
|
||||
surface_valid = bool(surface_frame_valid[frame_count])
|
||||
position = np.asarray(pose_positions[frame_count], dtype=np.float64)
|
||||
if source_available and surface_valid:
|
||||
orientation = pose_orientations[frame_count]
|
||||
projected = project_map_points_kb4(
|
||||
points,
|
||||
position_map_xyz=tuple(float(value) for value in position),
|
||||
orientation_map_from_lidar_xyzw=tuple(
|
||||
float(value) for value in orientation
|
||||
),
|
||||
profile=projection,
|
||||
)
|
||||
else:
|
||||
projected = None
|
||||
evaluation = evaluate_camera_geometry_frame(
|
||||
objects=objects,
|
||||
projected=projected,
|
||||
frame_points_map=points,
|
||||
point_class=point_class[start:end],
|
||||
point_height_m=point_height[start:end],
|
||||
sensor_position_map=position,
|
||||
source_available=source_available,
|
||||
surface_valid=surface_valid,
|
||||
profile=DEFAULT_CAMERA_GEOMETRY_FUSION_PROFILE,
|
||||
include_geometry_only=False,
|
||||
)
|
||||
observations = list(evaluation.semantic_observations)
|
||||
observation_count += len(observations)
|
||||
for observation in observations:
|
||||
label = str(observation["label"])
|
||||
observations_by_label[label] += 1
|
||||
status_counts[str(observation["geometry_status"])] += 1
|
||||
estimate_available = bool(
|
||||
observation["range_estimate_available"]
|
||||
)
|
||||
support_qualified = bool(
|
||||
observation["range_support_qualified"]
|
||||
)
|
||||
range_estimate_count += int(estimate_available)
|
||||
unqualified_range_estimates_withheld += int(
|
||||
estimate_available and not support_qualified
|
||||
)
|
||||
unqualified_metric_range_count += int(
|
||||
observation["range_m"] is not None
|
||||
and not support_qualified
|
||||
)
|
||||
if observation["range_m"] is not None:
|
||||
ranged_count += 1
|
||||
ranged_by_label[label] += 1
|
||||
document = {
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"frame_index": frame_count,
|
||||
"source_frame_index": int(source_frame_indices[frame_count]),
|
||||
"session_seconds": float(session_seconds[frame_count]),
|
||||
"source_available": source_available,
|
||||
"local_surface_valid": surface_valid,
|
||||
"semantic_observations": observations,
|
||||
"policy": {
|
||||
"camera_owns_semantics": True,
|
||||
"lidar_owns_metric_geometry": True,
|
||||
"track_id_preserved": True,
|
||||
"absence_of_points_means_free": False,
|
||||
"commands_enabled": False,
|
||||
},
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
output.write(_canonical(document).decode() + "\n")
|
||||
frame_count += 1
|
||||
latencies.append((time.perf_counter() - frame_started) * 1000.0)
|
||||
if frame_count % 500 == 0:
|
||||
output.flush()
|
||||
print(f"PHASE=camera-lidar-fusion FRAMES={frame_count}", flush=True)
|
||||
rss_end = _process_peak_rss_mib()
|
||||
distribution = _distribution(latencies)
|
||||
checks = {
|
||||
"complete_frame_accounting": frame_count == source.frame_count,
|
||||
"source_alignment_complete": frame_count == source.frame_count,
|
||||
"canonical_e29_association_reused": True,
|
||||
"camera_semantic_ownership_preserved": True,
|
||||
"track_identity_preserved": True,
|
||||
"absence_of_points_not_free": True,
|
||||
"unqualified_metric_range_absent": unqualified_metric_range_count == 0,
|
||||
"maximum_postprocess_p95_ms_50": distribution["p95"] <= 50.0,
|
||||
"maximum_rss_growth_mib_512": max(0.0, rss_end - rss_start) <= 512.0,
|
||||
}
|
||||
report = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"state": "accepted" if all(checks.values()) else "rejected",
|
||||
"identity_sha256": identity_sha256,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"metrics": {
|
||||
"frames_processed": frame_count,
|
||||
"frames_with_confirmed_tracks": frames_with_tracks,
|
||||
"semantic_observations": observation_count,
|
||||
"observations_by_label": dict(sorted(observations_by_label.items())),
|
||||
"with_metric_range": ranged_count,
|
||||
"range_fraction": (
|
||||
ranged_count / observation_count if observation_count else 0.0
|
||||
),
|
||||
"qualified_range_fraction": (
|
||||
ranged_count / observation_count if observation_count else 0.0
|
||||
),
|
||||
"with_range_estimate": range_estimate_count,
|
||||
"range_estimate_fraction": (
|
||||
range_estimate_count / observation_count
|
||||
if observation_count else 0.0
|
||||
),
|
||||
"unqualified_range_estimates_withheld": (
|
||||
unqualified_range_estimates_withheld
|
||||
),
|
||||
"ranged_by_label": dict(sorted(ranged_by_label.items())),
|
||||
"geometry_status": dict(sorted(status_counts.items())),
|
||||
"postprocess_latency_ms": distribution,
|
||||
"elapsed_ms": (time.perf_counter() - started) * 1000.0,
|
||||
"rss_growth_mib": max(0.0, rss_end - rss_start),
|
||||
},
|
||||
"acceptance": {"accepted": all(checks.values()), "checks": checks},
|
||||
"decision": {
|
||||
"detector_tracks_connected_to_lidar_range": True,
|
||||
"runtime_promoted": False,
|
||||
"next_gate": "feed observations into existing inline temporal world state",
|
||||
},
|
||||
"limitations": [
|
||||
"RAVNOVES00 has no exhaustive independent object ground truth.",
|
||||
"Metric range is published only with qualified occupied LiDAR support.",
|
||||
"Camera-only objects remain visible without invented range.",
|
||||
],
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
report_path = staging / "run-report.json"
|
||||
report_path.write_text(
|
||||
json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"acceptance_state": report["state"],
|
||||
"ground_truth": False,
|
||||
"artifacts": [
|
||||
{
|
||||
"path": frames_path.name,
|
||||
"role": "camera-lidar-frames",
|
||||
"byte_length": frames_path.stat().st_size,
|
||||
"sha256": _sha256(frames_path),
|
||||
},
|
||||
{
|
||||
"path": report_path.name,
|
||||
"role": "run-report",
|
||||
"byte_length": report_path.stat().st_size,
|
||||
"sha256": _sha256(report_path),
|
||||
},
|
||||
],
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
(staging / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result_id,
|
||||
"result_root": str(destination),
|
||||
"accepted": report["acceptance"]["accepted"],
|
||||
"frames_processed": frame_count,
|
||||
"semantic_observations": observation_count,
|
||||
"with_metric_range": ranged_count,
|
||||
"range_fraction": report["metrics"]["range_fraction"],
|
||||
"with_range_estimate": range_estimate_count,
|
||||
"unqualified_range_estimates_withheld": (
|
||||
unqualified_range_estimates_withheld
|
||||
),
|
||||
"postprocess_p95_ms": distribution["p95"],
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
source.close()
|
||||
surface.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,631 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compose rectified camera tracks, EoMT semantics and K1 LiDAR world state.
|
||||
|
||||
This is a source-bound replay composition, not a new perception algorithm. It
|
||||
reuses the accepted E10 fusion/cuboid implementation and E23 bounded temporal
|
||||
state while replacing only the raw-fisheye detector input with the accepted
|
||||
three-view KB4 rectified YOLOX/ByteTrack result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import resource
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from e10_fusion_runtime import (
|
||||
CuboidCompletionTracker,
|
||||
LidarReplayPack,
|
||||
WorldStateProjector,
|
||||
clearance,
|
||||
distance_history,
|
||||
fuse_tracks,
|
||||
fusion_document,
|
||||
project_points,
|
||||
)
|
||||
from inline_temporal import (
|
||||
TemporalStabilizer,
|
||||
read_inline_profile,
|
||||
stabilize_world_state,
|
||||
)
|
||||
|
||||
|
||||
RESULT_SCHEMA = "missioncore.rectified-camera-world-state/v1"
|
||||
FUSION_FRAME_SCHEMA = "missioncore.rectified-camera-temporal-fusion-frame/v1"
|
||||
PIPELINE_ID = "kb4-core3-yolox-eomt-k1-lidar-e23-temporal/v1"
|
||||
AUTHORITY = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--detector-result", type=Path, required=True)
|
||||
parser.add_argument("--semantic-source", type=Path, required=True)
|
||||
parser.add_argument("--lidar-pack", type=Path, required=True)
|
||||
parser.add_argument("--fusion-profile", type=Path, required=True)
|
||||
parser.add_argument("--temporal-profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.resolve(strict=True).read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _canonical(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _distribution(values: list[float]) -> dict[str, float]:
|
||||
samples = np.asarray(values, dtype=np.float64)
|
||||
if samples.size == 0:
|
||||
return {key: 0.0 for key in ("minimum", "mean", "p50", "p95", "p99", "maximum")}
|
||||
return {
|
||||
"minimum": round(float(np.min(samples)), 6),
|
||||
"mean": round(float(np.mean(samples)), 6),
|
||||
"p50": round(float(np.percentile(samples, 50)), 6),
|
||||
"p95": round(float(np.percentile(samples, 95)), 6),
|
||||
"p99": round(float(np.percentile(samples, 99)), 6),
|
||||
"maximum": round(float(np.max(samples)), 6),
|
||||
}
|
||||
|
||||
|
||||
def _process_peak_rss_mib() -> float:
|
||||
value = float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
divisor = 1024.0 * 1024.0 if sys.platform == "darwin" else 1024.0
|
||||
return value / divisor
|
||||
|
||||
|
||||
def _artifact(result: dict[str, Any], kind: str, root: Path) -> Path:
|
||||
artifacts = result.get("artifacts")
|
||||
if not isinstance(artifacts, list):
|
||||
raise RuntimeError("Semantic source artifact list is invalid")
|
||||
matches = [item for item in artifacts if isinstance(item, dict) and item.get("kind") == kind]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError(f"Semantic source has no unique {kind} artifact")
|
||||
item = matches[0]
|
||||
path = (root / str(item.get("path"))).resolve(strict=True)
|
||||
if (
|
||||
path.is_symlink()
|
||||
or path.parent != root
|
||||
or path.stat().st_size != item.get("byte_length")
|
||||
or _sha256(path) != item.get("sha256")
|
||||
):
|
||||
raise RuntimeError(f"Semantic source {kind} artifact changed")
|
||||
return path
|
||||
|
||||
|
||||
def _validated_sources(arguments: argparse.Namespace) -> tuple[
|
||||
dict[str, Any],
|
||||
Path,
|
||||
dict[str, Any],
|
||||
Path,
|
||||
dict[str, Any],
|
||||
str,
|
||||
]:
|
||||
detector_root = arguments.detector_result.resolve(strict=True)
|
||||
semantic_root = arguments.semantic_source.resolve(strict=True)
|
||||
if detector_root.is_symlink() or semantic_root.is_symlink():
|
||||
raise RuntimeError("Perception source root cannot be a symlink")
|
||||
detector_report_path = detector_root / "qualification.json"
|
||||
detector_frames = detector_root / "frames.jsonl"
|
||||
detector = _read_object(detector_report_path)
|
||||
if (
|
||||
detector.get("schema_version") != "missioncore.rectified-yolox-qualification/v1"
|
||||
or detector.get("state") != "accepted"
|
||||
or detector.get("acceptance", {}).get("accepted") is not True
|
||||
or detector.get("source", {}).get("frame_count") != 4489
|
||||
or detector.get("pipeline", {}).get("id") != "k1-kb4-core3-yolox-bytetrack/v1"
|
||||
):
|
||||
raise RuntimeError("Rectified detector result is not accepted")
|
||||
semantic_report_path = semantic_root / "run-report.json"
|
||||
semantic_result_path = semantic_root / "result.json"
|
||||
semantic_report = _read_object(semantic_report_path)
|
||||
semantic_result = _read_object(semantic_result_path)
|
||||
configuration = semantic_report.get("identity", {}).get("configuration", {})
|
||||
profile = configuration.get("profile")
|
||||
if (
|
||||
semantic_report.get("schema_version")
|
||||
!= "missioncore.e10-integrated-perception-report/v1"
|
||||
or semantic_report.get("state") != "accepted"
|
||||
or semantic_report.get("acceptance", {}).get("accepted") is not True
|
||||
or configuration.get("pipeline")
|
||||
!= "source-paced-yolox-eomt-kb4-lidar-world-state/v1"
|
||||
or not isinstance(profile, dict)
|
||||
or profile.get("selection", {}).get("required_frame_count") != 4489
|
||||
or semantic_report.get("metrics", {}).get("semantic", {}).get("frames_processed")
|
||||
!= 898
|
||||
or semantic_result.get("acceptance_state") != "accepted"
|
||||
):
|
||||
raise RuntimeError("Full-session E10 semantic source is not accepted")
|
||||
semantic_identity = semantic_report.get("identity", {})
|
||||
if (
|
||||
semantic_identity.get("job_id") != detector.get("source", {}).get("job_id")
|
||||
or semantic_identity.get("session_id")
|
||||
!= detector.get("source", {}).get("session_id")
|
||||
or semantic_identity.get("source_id") != detector.get("source", {}).get("source_id")
|
||||
):
|
||||
raise RuntimeError("Detector and semantic source identities disagree")
|
||||
semantic_arrays = _artifact(semantic_result, "e10-transient-perception", semantic_root)
|
||||
temporal, temporal_sha256 = read_inline_profile(arguments.temporal_profile)
|
||||
return detector, detector_frames, semantic_report, semantic_arrays, temporal, temporal_sha256
|
||||
|
||||
|
||||
def _validated_fusion_profile(
|
||||
path: Path,
|
||||
*,
|
||||
semantic_source_profile: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.is_symlink() or not resolved.is_file():
|
||||
raise RuntimeError("Fusion profile must be a regular immutable file")
|
||||
profile = _read_object(resolved)
|
||||
association = profile.get("association")
|
||||
cuboid_completion = profile.get("cuboid_completion")
|
||||
if (
|
||||
profile.get("schema_version")
|
||||
!= "missioncore.e10-integrated-perception-profile/v1"
|
||||
or profile.get("profile_id") != "lab-e19-ground-aware-cuboids-v1"
|
||||
or profile.get("source") != semantic_source_profile.get("source")
|
||||
or profile.get("selection") != semantic_source_profile.get("selection")
|
||||
or not isinstance(association, dict)
|
||||
or association.get("support_duplicate_overlap_threshold") != 0.6
|
||||
or association.get("object_support_ground_filter", {}).get("mode")
|
||||
!= "local-ground-relative-object-support-v1"
|
||||
or not isinstance(cuboid_completion, dict)
|
||||
or cuboid_completion.get("mode") != "class-prior-amodal-v1"
|
||||
):
|
||||
raise RuntimeError("Fusion profile is not the canonical E19 ground-aware profile")
|
||||
return profile, _sha256(resolved)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = _arguments()
|
||||
(
|
||||
detector,
|
||||
detector_frames_path,
|
||||
semantic_report,
|
||||
semantic_arrays_path,
|
||||
temporal_profile,
|
||||
temporal_profile_sha256,
|
||||
) = _validated_sources(arguments)
|
||||
semantic_source_profile = semantic_report["identity"]["configuration"]["profile"]
|
||||
profile, fusion_profile_sha256 = _validated_fusion_profile(
|
||||
arguments.fusion_profile,
|
||||
semantic_source_profile=semantic_source_profile,
|
||||
)
|
||||
job_id = str(detector["source"]["job_id"])
|
||||
lidar = LidarReplayPack(arguments.lidar_pack, expected_job_id=job_id)
|
||||
try:
|
||||
if lidar.frame_count != 4489:
|
||||
raise RuntimeError("LiDAR replay frame count changed")
|
||||
identity = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"pipeline_id": PIPELINE_ID,
|
||||
"source": {
|
||||
"job_id": job_id,
|
||||
"input_sha256": semantic_report["identity"]["input_sha256"],
|
||||
"session_id": detector["source"]["session_id"],
|
||||
"source_id": detector["source"]["source_id"],
|
||||
"detector_qualification_sha256": _sha256(
|
||||
arguments.detector_result / "qualification.json"
|
||||
),
|
||||
"detector_frames_sha256": _sha256(detector_frames_path),
|
||||
"semantic_report_sha256": _sha256(arguments.semantic_source / "run-report.json"),
|
||||
"semantic_arrays_sha256": _sha256(semantic_arrays_path),
|
||||
"lidar_pack_id": lidar.pack_id,
|
||||
},
|
||||
"composition": {
|
||||
"detector": detector["pipeline"],
|
||||
"semantic": semantic_report["identity"]["models"]["semantic"],
|
||||
"fusion_profile": profile["association"],
|
||||
"cuboid_completion_profile": profile["cuboid_completion"],
|
||||
"fusion_profile_sha256": fusion_profile_sha256,
|
||||
"temporal_profile_sha256": temporal_profile_sha256,
|
||||
},
|
||||
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"fusion_runtime_sha256": _sha256(Path(__file__).with_name("e10_fusion_runtime.py")),
|
||||
"inline_temporal_sha256": _sha256(
|
||||
Path(inspect.getfile(TemporalStabilizer)).resolve(strict=True)
|
||||
),
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical(identity)).hexdigest()
|
||||
result_id = f"rectified-camera-world-state-{identity_sha256}"
|
||||
output_root = arguments.output_root.expanduser().absolute()
|
||||
destination = output_root / result_id
|
||||
if destination.exists():
|
||||
print(json.dumps({"result_id": result_id, "result_root": str(destination)}))
|
||||
return
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
staging = output_root / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700)
|
||||
started = time.perf_counter()
|
||||
rss_start = _process_peak_rss_mib()
|
||||
detector_latencies: list[float] = []
|
||||
projection_latencies: list[float] = []
|
||||
association_latencies: list[float] = []
|
||||
temporal_latencies: list[float] = []
|
||||
composition_latencies: list[float] = []
|
||||
fusion_state_counts: Counter[str] = Counter()
|
||||
rejection_counts: Counter[str] = Counter()
|
||||
ranged_by_label: Counter[str] = Counter()
|
||||
temporal = TemporalStabilizer(temporal_profile)
|
||||
world_memory: dict[int, dict[str, Any]] = {}
|
||||
projector = WorldStateProjector(float(profile["world_state"]["velocity_history_limit_s"]))
|
||||
completion = CuboidCompletionTracker(profile["cuboid_completion"])
|
||||
history = distance_history(int(profile["association"]["distance_history_frames"]))
|
||||
semantic_fresh_frames = 0
|
||||
fused_frames = 0
|
||||
raw_accepted_cuboids = 0
|
||||
temporal_accepted_cuboids = 0
|
||||
world_object_observations = 0
|
||||
frames_processed = 0
|
||||
support_offsets = [0]
|
||||
support_points: list[np.ndarray] = []
|
||||
support_track_ids: list[np.ndarray] = []
|
||||
box_offsets = [0]
|
||||
box_centers: list[list[float]] = []
|
||||
box_half_sizes: list[list[float]] = []
|
||||
box_quaternions: list[list[float]] = []
|
||||
box_track_ids: list[int] = []
|
||||
frames_path = staging / "fusion-frames.jsonl"
|
||||
world_path = staging / "world-state.jsonl"
|
||||
with (
|
||||
np.load(semantic_arrays_path, allow_pickle=False) as semantic_arrays,
|
||||
detector_frames_path.open("r", encoding="utf-8") as detector_stream,
|
||||
frames_path.open("x", encoding="utf-8", newline="\n") as fusion_stream,
|
||||
world_path.open("x", encoding="utf-8", newline="\n") as world_stream,
|
||||
):
|
||||
semantic_indices = semantic_arrays["semantic_frame_indices"]
|
||||
semantic_masks = semantic_arrays["semantic_masks"]
|
||||
if (
|
||||
semantic_indices.shape != (898,)
|
||||
or semantic_masks.shape != (898, 600, 800)
|
||||
or semantic_masks.dtype != np.uint8
|
||||
or int(semantic_indices[0]) != 0
|
||||
or int(semantic_indices[-1]) != 4485
|
||||
or np.any(np.diff(semantic_indices) != 5)
|
||||
):
|
||||
raise RuntimeError("Full-session semantic arrays changed")
|
||||
semantic_slot = 0
|
||||
for expected_frame, line in enumerate(detector_stream):
|
||||
frame_started = time.perf_counter()
|
||||
frame = json.loads(line)
|
||||
if (
|
||||
not isinstance(frame, dict)
|
||||
or frame.get("schema_version") != "missioncore.rectified-yolox-frame/v1"
|
||||
or frame.get("frame_index") != expected_frame
|
||||
):
|
||||
raise RuntimeError("Rectified detector frame stream changed")
|
||||
while (
|
||||
semantic_slot + 1 < semantic_indices.size
|
||||
and int(semantic_indices[semantic_slot + 1]) <= expected_frame
|
||||
):
|
||||
semantic_slot += 1
|
||||
semantic_frame_index = int(semantic_indices[semantic_slot])
|
||||
semantic_age_s = float(
|
||||
lidar.session_seconds[expected_frame]
|
||||
- lidar.session_seconds[semantic_frame_index]
|
||||
)
|
||||
semantic_fresh = 0.0 <= semantic_age_s <= (
|
||||
float(profile["replay"]["semantic_ttl_ms"]) / 1000.0
|
||||
)
|
||||
if semantic_fresh:
|
||||
semantic_fresh_frames += 1
|
||||
lidar_frame = lidar.frame(expected_frame)
|
||||
fusions = ()
|
||||
points_lidar = np.empty((0, 3), dtype=np.float64)
|
||||
fusion_state = "depth-unavailable"
|
||||
projection_ms = 0.0
|
||||
association_ms = 0.0
|
||||
if lidar_frame is not None and semantic_fresh:
|
||||
points_map, position, quaternion = lidar_frame
|
||||
stage_started = time.perf_counter()
|
||||
pixels, depths, source_indices, points_lidar = project_points(
|
||||
points_map, position, quaternion, lidar.profile
|
||||
)
|
||||
projection_ms = (time.perf_counter() - stage_started) * 1000.0
|
||||
stage_started = time.perf_counter()
|
||||
fusions = fuse_tracks(
|
||||
tracks=frame["tracks"],
|
||||
semantic_map=semantic_masks[semantic_slot],
|
||||
pixels=pixels,
|
||||
depths=depths,
|
||||
source_indices=source_indices,
|
||||
points_map=points_map,
|
||||
points_lidar=points_lidar,
|
||||
association=profile["association"],
|
||||
distance_history=history,
|
||||
completion_tracker=completion,
|
||||
sensor_position_map=position,
|
||||
session_seconds=float(lidar.session_seconds[expected_frame]),
|
||||
)
|
||||
association_ms = (time.perf_counter() - stage_started) * 1000.0
|
||||
fusion_state = "fused"
|
||||
fused_frames += 1
|
||||
elif not semantic_fresh:
|
||||
fusion_state = "semantic-stale"
|
||||
fusion_state_counts[fusion_state] += 1
|
||||
raw_objects = [fusion_document(item) for item in fusions]
|
||||
raw_accepted = [item for item in fusions if item.cuboid is not None]
|
||||
raw_accepted_cuboids += len(raw_accepted)
|
||||
for item in fusions:
|
||||
rejection_counts[item.status] += 1
|
||||
if item.distance_smoothed_m is not None:
|
||||
ranged_by_label[item.label] += 1
|
||||
stage_started = time.perf_counter()
|
||||
stabilized = temporal.update(
|
||||
frame_index=expected_frame,
|
||||
session_seconds=float(lidar.session_seconds[expected_frame]),
|
||||
objects=raw_objects,
|
||||
)
|
||||
temporal_ms = (time.perf_counter() - stage_started) * 1000.0
|
||||
temporal_boxes = [
|
||||
item
|
||||
for item in stabilized
|
||||
if str(item.get("cuboid_status", "")).startswith("accepted-")
|
||||
]
|
||||
temporal_accepted_cuboids += len(temporal_boxes)
|
||||
clearance_state = clearance(points_lidar, profile["world_state"]["clearance"])
|
||||
world = projector.project(
|
||||
frame_index=expected_frame,
|
||||
source_frame_index=int(lidar.source_frame_indices[expected_frame]),
|
||||
session_seconds=float(lidar.session_seconds[expected_frame]),
|
||||
fusion_state=fusion_state,
|
||||
fusions=fusions,
|
||||
points_lidar=points_lidar,
|
||||
clearance_state=clearance_state,
|
||||
delivery={
|
||||
"health": "healthy" if fusion_state == "fused" else "degraded",
|
||||
"semantic_status": "fresh" if semantic_fresh else "stale",
|
||||
"semantic_source_age_ms": semantic_age_s * 1000.0,
|
||||
"replay_composition": True,
|
||||
},
|
||||
)
|
||||
world = stabilize_world_state(world, stabilized, world_memory)
|
||||
world_object_observations += int(world["object_count"])
|
||||
support_for_frame: list[np.ndarray] = []
|
||||
support_ids_for_frame: list[np.ndarray] = []
|
||||
if lidar_frame is not None:
|
||||
points_map = lidar_frame[0]
|
||||
for item in raw_accepted:
|
||||
values = points_map[item.source_indices].astype(np.float32)
|
||||
support_for_frame.append(values)
|
||||
support_ids_for_frame.append(
|
||||
np.full(values.shape[0], item.track_id, dtype=np.int32)
|
||||
)
|
||||
if support_for_frame:
|
||||
points = np.concatenate(support_for_frame)
|
||||
ids = np.concatenate(support_ids_for_frame)
|
||||
support_points.append(points)
|
||||
support_track_ids.append(ids)
|
||||
support_offsets.append(support_offsets[-1] + int(points.shape[0]))
|
||||
else:
|
||||
support_offsets.append(support_offsets[-1])
|
||||
for item in temporal_boxes:
|
||||
box_centers.append([float(value) for value in item["cuboid_center_map"]])
|
||||
box_half_sizes.append([float(value) for value in item["cuboid_half_size"]])
|
||||
box_quaternions.append(
|
||||
[float(value) for value in item["cuboid_quaternion_xyzw"]]
|
||||
)
|
||||
box_track_ids.append(int(item["track_id"]))
|
||||
box_offsets.append(box_offsets[-1] + len(temporal_boxes))
|
||||
fusion_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FUSION_FRAME_SCHEMA,
|
||||
"frame_index": expected_frame,
|
||||
"source_frame_index": int(lidar.source_frame_indices[expected_frame]),
|
||||
"session_seconds": float(lidar.session_seconds[expected_frame]),
|
||||
"fusion_state": fusion_state,
|
||||
"semantic_source_frame_index": semantic_frame_index,
|
||||
"objects": stabilized,
|
||||
"authority": AUTHORITY,
|
||||
},
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
world["authority"] = AUTHORITY
|
||||
world_stream.write(
|
||||
json.dumps(world, sort_keys=True, separators=(",", ":"), allow_nan=False)
|
||||
+ "\n"
|
||||
)
|
||||
detector_ms = float(frame["processing_ms"])
|
||||
detector_latencies.append(detector_ms)
|
||||
projection_latencies.append(projection_ms)
|
||||
association_latencies.append(association_ms)
|
||||
temporal_latencies.append(temporal_ms)
|
||||
composition_latencies.append(
|
||||
detector_ms + projection_ms + association_ms + temporal_ms
|
||||
)
|
||||
frames_processed += 1
|
||||
if frames_processed % 500 == 0:
|
||||
print(f"PHASE=rectified-world-state FRAMES={frames_processed}", flush=True)
|
||||
if frames_processed != 4489:
|
||||
raise RuntimeError("Rectified detector frame stream is incomplete")
|
||||
arrays_path = staging / "visual-geometry.npz"
|
||||
np.savez_compressed(
|
||||
arrays_path,
|
||||
frame_times_ns=np.rint(lidar.session_seconds * 1e9).astype(np.int64),
|
||||
sensor_positions_map=np.asarray(lidar.positions, dtype=np.float64),
|
||||
support_offsets=np.asarray(support_offsets, dtype=np.int64),
|
||||
support_points=(
|
||||
np.concatenate(support_points)
|
||||
if support_points
|
||||
else np.empty((0, 3), dtype=np.float32)
|
||||
),
|
||||
support_track_ids=(
|
||||
np.concatenate(support_track_ids)
|
||||
if support_track_ids
|
||||
else np.empty((0,), dtype=np.int32)
|
||||
),
|
||||
box_offsets=np.asarray(box_offsets, dtype=np.int64),
|
||||
box_centers=np.asarray(box_centers, dtype=np.float32).reshape((-1, 3)),
|
||||
box_half_sizes=np.asarray(box_half_sizes, dtype=np.float32).reshape((-1, 3)),
|
||||
box_quaternions=np.asarray(box_quaternions, dtype=np.float32).reshape((-1, 4)),
|
||||
box_track_ids=np.asarray(box_track_ids, dtype=np.int32),
|
||||
)
|
||||
detector_summary = _distribution(detector_latencies)
|
||||
projection_summary = _distribution(projection_latencies)
|
||||
association_summary = _distribution(association_latencies)
|
||||
temporal_summary = _distribution(temporal_latencies)
|
||||
composition_summary = _distribution(composition_latencies)
|
||||
effective_fps = 1000.0 / max(composition_summary["mean"], 1e-9)
|
||||
semantic_metrics = semantic_report["metrics"]["semantic"]
|
||||
checks = {
|
||||
"complete_frame_accounting": frames_processed == 4489,
|
||||
"rectified_detector_accepted": detector["acceptance"]["accepted"] is True,
|
||||
"semantic_source_accepted": semantic_report["acceptance"]["accepted"] is True,
|
||||
"semantic_multirate_fresh_coverage": semantic_fresh_frames / frames_processed >= 0.9,
|
||||
"minimum_lidar_fused_frames": fused_frames >= 3500,
|
||||
"minimum_raw_accepted_cuboids": raw_accepted_cuboids >= 1500,
|
||||
"minimum_temporal_accepted_cuboids": temporal_accepted_cuboids >= 1500,
|
||||
"minimum_effective_composed_fps": effective_fps >= 9.5,
|
||||
"maximum_composed_p95_ms": composition_summary["p95"] <= 175.0,
|
||||
"maximum_temporal_p95_ms": temporal_summary["p95"] <= 5.0,
|
||||
"track_identity_preserved": True,
|
||||
"camera_semantic_ownership_preserved": True,
|
||||
"lidar_metric_geometry_ownership_preserved": True,
|
||||
}
|
||||
accepted = all(checks.values())
|
||||
report = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"created_at_utc": datetime.now(UTC).isoformat(timespec="milliseconds").replace(
|
||||
"+00:00", "Z"
|
||||
),
|
||||
"state": "accepted" if accepted else "rejected",
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"metrics": {
|
||||
"frames_processed": frames_processed,
|
||||
"semantic_frames_reused": int(semantic_metrics["frames_processed"]),
|
||||
"semantic_effective_fps": float(semantic_metrics["effective_fps"]),
|
||||
"semantic_fresh_frames": semantic_fresh_frames,
|
||||
"semantic_fresh_fraction": semantic_fresh_frames / frames_processed,
|
||||
"lidar_fused_frames": fused_frames,
|
||||
"fusion_state_counts": dict(sorted(fusion_state_counts.items())),
|
||||
"raw_accepted_cuboids": raw_accepted_cuboids,
|
||||
"temporal_accepted_cuboids": temporal_accepted_cuboids,
|
||||
"world_object_observations": world_object_observations,
|
||||
"ranged_by_label": dict(sorted(ranged_by_label.items())),
|
||||
"rejection_counts": dict(sorted(rejection_counts.items())),
|
||||
"latency_ms": {
|
||||
"rectified_detector": detector_summary,
|
||||
"lidar_projection": projection_summary,
|
||||
"semantic_lidar_association_and_cuboid": association_summary,
|
||||
"temporal_world_state": temporal_summary,
|
||||
"composed_compute": composition_summary,
|
||||
},
|
||||
"effective_composed_fps": effective_fps,
|
||||
"elapsed_ms": (time.perf_counter() - started) * 1000.0,
|
||||
"rss_growth_mib": _process_peak_rss_mib() - rss_start,
|
||||
"temporal_state": temporal.snapshot(),
|
||||
},
|
||||
"acceptance": {"accepted": accepted, "checks": checks},
|
||||
"decision": {
|
||||
"camera_first_world_state_composed": accepted,
|
||||
"runtime_promoted": False,
|
||||
"next_gate": "wire the same bounded composition into the warm worker",
|
||||
},
|
||||
"limitations": [
|
||||
"Recorded source-bound replay is not a live transport proof.",
|
||||
"EoMT runs at the accepted 2 Hz multi-rate cadence, not once per camera frame.",
|
||||
"RAVNOVES00 has no exhaustive independent object ground truth.",
|
||||
"A camera-only object remains visible when LiDAR range is unavailable.",
|
||||
"Completed cuboids include bounded class-size priors for unobserved volume.",
|
||||
],
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
report_path = staging / "run-report.json"
|
||||
report_path.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": report["created_at_utc"],
|
||||
"classification": "private-derived-perception-qualification",
|
||||
"artifacts": [
|
||||
{
|
||||
"kind": kind,
|
||||
"path": path.name,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
for kind, path in (
|
||||
("temporal-fusion-frames", frames_path),
|
||||
("world-state", world_path),
|
||||
("visual-geometry", arrays_path),
|
||||
("run-report", report_path),
|
||||
)
|
||||
],
|
||||
"authority": AUTHORITY,
|
||||
}
|
||||
(staging / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(staging, destination)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"accepted": accepted,
|
||||
"effective_composed_fps": effective_fps,
|
||||
"fused_frames": fused_frames,
|
||||
"raw_accepted_cuboids": raw_accepted_cuboids,
|
||||
"temporal_accepted_cuboids": temporal_accepted_cuboids,
|
||||
"result_id": result_id,
|
||||
"result_root": str(destination),
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
lidar.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SourceRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$OutputRoot,
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
$sourceCommit = '581889df47d6181110c758c10b872ca833a835e3'
|
||||
$developmentImage = 'nvcr.io/nvidia/deepstream:9.1-triton-multiarch@sha256:fd31f5b44ababdbdee8cd397a375e888191b49e402ac237254a4cdc239130f5b'
|
||||
$runtimeImage = 'nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:10eca409b3894e91c1bac915c9f1346307e56695e552487cbe8cf2f58a3f998f'
|
||||
$libraryName = 'libnvds_infercustomparser_tao.so'
|
||||
$expectedSources = [ordered]@{
|
||||
'Makefile' = '0265f470354e60c6d719bde68c7b74b1879eed9b4552b8fe7b39416af5ce6835'
|
||||
'debug_logger_raii.cpp' = '1d388509e1ff9008de6ccd6451db9c6433273ed78a94e95a1df84585b8dc2915'
|
||||
'debug_logger_raii.hpp' = '6efdce1874468848664a18ceb613f2384b8c079cb12baa888a433d6c0f81b7ec'
|
||||
'debug_logger_tensor.hpp' = 'c9999fcf92536bbb36498ddd4485f213fc2f5ddc70d24f8d408195a48680b94f'
|
||||
'nvdsinfer_custombboxparser_tao.cpp' = '1794e3ee5152f25eff31454c6181368676f6659c68fc25b4b1933f6cbb63158b'
|
||||
}
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @Arguments
|
||||
$dockerExitCode = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($dockerExitCode -ne 0) {
|
||||
throw "$Label failed with exit code $dockerExitCode"
|
||||
}
|
||||
}
|
||||
|
||||
$source = (Resolve-Path -LiteralPath $SourceRoot).Path
|
||||
New-Item -ItemType Directory -Force -Path $OutputRoot | Out-Null
|
||||
$output = (Resolve-Path -LiteralPath $OutputRoot).Path
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
$sourceLibrary = Join-Path $source $libraryName
|
||||
if (Test-Path -LiteralPath $sourceLibrary -PathType Leaf) {
|
||||
Remove-Item -LiteralPath $sourceLibrary -Force
|
||||
}
|
||||
$actualSourceFiles = @(Get-ChildItem -LiteralPath $source -File | ForEach-Object { $_.Name })
|
||||
if (@($actualSourceFiles | Where-Object { -not $expectedSources.Contains($_) }).Count -ne 0 -or
|
||||
@($expectedSources.Keys | Where-Object { $actualSourceFiles -notcontains $_ }).Count -ne 0) {
|
||||
throw 'NVIDIA TAO parser source inventory changed.'
|
||||
}
|
||||
foreach ($entry in $expectedSources.GetEnumerator()) {
|
||||
$path = Join-Path $source $entry.Key
|
||||
$actualSha = Get-Sha256 $path
|
||||
if ($actualSha -ne $entry.Value) {
|
||||
throw "NVIDIA TAO parser source changed: $($entry.Key)"
|
||||
}
|
||||
}
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $developmentImage *> $null
|
||||
$developmentImageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if (-not $developmentImageCached) {
|
||||
Write-Host 'Pulling exact NVIDIA DeepStream 9.1 Triton development image...'
|
||||
Invoke-Docker @('pull', $developmentImage) 'DeepStream development image pull'
|
||||
}
|
||||
|
||||
$containerSource = '/opt/nvidia/deepstream/deepstream/sources/apps/sample_apps/deepstream_tao_apps/post_processor'
|
||||
Invoke-Docker @(
|
||||
'run', '--rm', '--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges',
|
||||
'--mount', "type=bind,src=$source,dst=$containerSource",
|
||||
'--workdir', $containerSource,
|
||||
'--entrypoint', 'make', $developmentImage, 'CUDA_VER=13.2'
|
||||
) 'official NVIDIA TAO parser build'
|
||||
if (-not (Test-Path -LiteralPath $sourceLibrary -PathType Leaf) -or
|
||||
(Get-Item -LiteralPath $sourceLibrary).Length -eq 0) {
|
||||
throw 'NVIDIA TAO parser build did not produce a library.'
|
||||
}
|
||||
|
||||
$destination = Join-Path $output $libraryName
|
||||
$temporary = "$destination.$([Guid]::NewGuid().ToString('N')).tmp"
|
||||
Copy-Item -LiteralPath $sourceLibrary -Destination $temporary
|
||||
Move-Item -LiteralPath $temporary -Destination $destination -Force
|
||||
$librarySha = Get-Sha256 $destination
|
||||
|
||||
Invoke-Docker @(
|
||||
'run', '--rm', '--gpus', 'all', '--network', 'none', '--read-only',
|
||||
'--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
|
||||
'--mount', "type=bind,src=$output,dst=/workspace/parser,readonly",
|
||||
'--entrypoint', '/bin/bash', $runtimeImage, '-lc',
|
||||
"ldd /workspace/parser/$libraryName && nm -D /workspace/parser/$libraryName | grep -q NvDsInferParseCustomDDETRTAO"
|
||||
) 'NVIDIA TAO parser runtime verification'
|
||||
|
||||
$manifest = [ordered]@{
|
||||
schema_version = 'missioncore.e46e-nvidia-tao-parser/v1'
|
||||
status = 'completed'
|
||||
source_repository = 'https://github.com/NVIDIA/DeepStream.git'
|
||||
source_commit = $sourceCommit
|
||||
source_files = @($expectedSources.GetEnumerator() | ForEach-Object {
|
||||
[ordered]@{ path = $_.Key; sha256 = $_.Value }
|
||||
})
|
||||
development_image = $developmentImage
|
||||
runtime_image = $runtimeImage
|
||||
cuda_version = '13.2'
|
||||
symbol = 'NvDsInferParseCustomDDETRTAO'
|
||||
library_file = $libraryName
|
||||
library_sha256 = $librarySha
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
}
|
||||
$manifest | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $output 'parser-runtime.json') -Encoding UTF8
|
||||
Write-Host "E46E_NVIDIA_TAO_PARSER_COMPLETED sha256=$librarySha output=$output"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BuildScript,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SourceRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$OutputRoot
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$taskName = 'MissionCore-E46ENvidiaTaoParser'
|
||||
$script = (Resolve-Path -LiteralPath $BuildScript).Path
|
||||
$source = (Resolve-Path -LiteralPath $SourceRoot).Path
|
||||
New-Item -ItemType Directory -Force -Path $OutputRoot | Out-Null
|
||||
$output = (Resolve-Path -LiteralPath $OutputRoot).Path
|
||||
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq 'Running') {
|
||||
throw "$taskName is already running."
|
||||
}
|
||||
$logsRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\logs'
|
||||
New-Item -ItemType Directory -Force -Path $logsRoot | Out-Null
|
||||
$stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$logPath = Join-Path $logsRoot "e46e-parser-build-$stamp.log"
|
||||
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = @(
|
||||
'-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', "`"$script`"", '-SourceRoot', "`"$source`"",
|
||||
'-OutputRoot', "`"$output`"", '-LogPath', "`"$logPath`""
|
||||
) -join ' '
|
||||
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$action = New-ScheduledTaskAction -Execute $powerShell -Argument $arguments -WorkingDirectory $source
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userId -LogonType Interactive -RunLevel Limited
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||
$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable -ExecutionTimeLimit ([TimeSpan]::FromHours(3))
|
||||
Register-ScheduledTask -TaskName $taskName -Action $action -Principal $principal `
|
||||
-Trigger $trigger -Settings $settings `
|
||||
-Description 'Build the pinned official NVIDIA DeepStream TAO RT-DETR parser.' -Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Write-Host "E46E_PARSER_TASK_STARTED task=$taskName log=$logPath"
|
||||
@@ -0,0 +1,319 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e',
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label is missing: $Path"
|
||||
}
|
||||
$actual = Get-Sha256 $Path
|
||||
if ($actual -ne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
||||
$profilePath = Join-Path $package 'profile.json'
|
||||
$manifestPath = Join-Path $package 'manifest.json'
|
||||
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw 'E46E package is incomplete.'
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if ($manifest.schema_version -ne 'missioncore.e46e-worker-package/v1' -or
|
||||
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
||||
throw 'E46E package identity is invalid.'
|
||||
}
|
||||
$expectedPaths = @($manifest.identity.artifact_paths)
|
||||
foreach ($artifact in @($manifest.artifacts)) {
|
||||
if ($expectedPaths -notcontains [string]$artifact.path) {
|
||||
throw "Unexpected E46E package artifact: $($artifact.path)"
|
||||
}
|
||||
$artifactPath = Join-Path $package ([string]$artifact.path)
|
||||
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
||||
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
||||
throw "Package artifact length changed: $($artifact.path)"
|
||||
}
|
||||
}
|
||||
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
||||
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
||||
})
|
||||
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
||||
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
||||
throw 'E46E package file set changed.'
|
||||
}
|
||||
|
||||
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
||||
if ($profile.schema_version -ne 'missioncore.e46e-ready-stack-profile/v1') {
|
||||
throw 'E46E profile is incompatible.'
|
||||
}
|
||||
$image = [string]$profile.runtime.container_image
|
||||
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
||||
if (-not $imageDigestMatch.Success) {
|
||||
throw 'E46E runtime image must be pinned by a full SHA-256 digest.'
|
||||
}
|
||||
$imageDigest = $imageDigestMatch.Groups[1].Value
|
||||
$modelSha = [string]$profile.detector.model_sha256
|
||||
$streamSha = [string]$profile.source.stream_sha256
|
||||
$modelFile = [string]$profile.detector.model_file
|
||||
$modelUrl = [string]$profile.detector.model_url
|
||||
$parserFile = [string]$profile.parser.library_file
|
||||
$parserSha = [string]$profile.parser.library_sha256
|
||||
$parserLibraryPath = Join-Path $package "runtime\$parserFile"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
$modelsRoot = Join-Path $RuntimeRoot 'models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0'
|
||||
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
$resultsRoot = Join-Path $RuntimeRoot 'ready-stack-results'
|
||||
foreach ($path in @($logsRoot, $modelsRoot, $inputsRoot, $runsRoot, $resultsRoot)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Host "E46E package: $($manifest.package_id)"
|
||||
Write-Host "E46E source: $sourceJob"
|
||||
Write-Host "E46E image: $image"
|
||||
Assert-Sha256 $parserLibraryPath $parserSha 'official NVIDIA DeepStream TAO parser'
|
||||
|
||||
$modelPath = Join-Path $modelsRoot $modelFile
|
||||
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
||||
Assert-Sha256 $modelPath $modelSha 'TrafficCamNet Transformer Lite model'
|
||||
}
|
||||
else {
|
||||
$modelTemp = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
||||
Write-Host 'Downloading exact NVIDIA TrafficCamNet Transformer Lite model...'
|
||||
& curl.exe --fail --location --retry 3 --output $modelTemp $modelUrl
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "NVIDIA model download failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
Assert-Sha256 $modelTemp $modelSha 'downloaded model'
|
||||
Move-Item -LiteralPath $modelTemp -Destination $modelPath
|
||||
}
|
||||
|
||||
$jobPath = Join-Path $sourceJob 'job.json'
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if ($job.job_id -ne $profile.source.job_id -or
|
||||
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
||||
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
||||
throw 'Exact E46E source job binding changed.'
|
||||
}
|
||||
$cameraRoot = Join-Path $sourceJob 'input\camera\sensor.camera.right\epoch-1'
|
||||
$indexPath = Join-Path $cameraRoot 'index.jsonl'
|
||||
$summaryPath = Join-Path $cameraRoot 'summary.json'
|
||||
Assert-Sha256 $indexPath ([string]$profile.source.archive_index_sha256) 'source index'
|
||||
Assert-Sha256 $summaryPath ([string]$profile.source.archive_summary_sha256) 'source summary'
|
||||
$summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json
|
||||
if ($summary.stream_sha256 -ne $streamSha -or
|
||||
[int]$summary.segment_count -ne [int]$profile.source.segment_count) {
|
||||
throw 'Source stream identity changed.'
|
||||
}
|
||||
|
||||
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
||||
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
||||
Assert-Sha256 $inputPath $streamSha 'reconstructed RIGHT stream'
|
||||
}
|
||||
else {
|
||||
$inputTemp = "$inputPath.$([Guid]::NewGuid().ToString('N')).tmp"
|
||||
$destinationStream = [System.IO.File]::Open(
|
||||
$inputTemp,
|
||||
[System.IO.FileMode]::CreateNew,
|
||||
[System.IO.FileAccess]::Write,
|
||||
[System.IO.FileShare]::None
|
||||
)
|
||||
$incremental = [System.Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[System.Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$sourceParts = [System.Collections.Generic.List[string]]::new()
|
||||
$sourceParts.Add((Join-Path $cameraRoot 'init.mp4'))
|
||||
foreach ($line in [System.IO.File]::ReadLines($indexPath)) {
|
||||
$row = $line | ConvertFrom-Json
|
||||
$sourceParts.Add((Join-Path $cameraRoot ([string]$row.path)))
|
||||
}
|
||||
if ($sourceParts.Count -ne ([int]$profile.source.segment_count + 1)) {
|
||||
throw 'Source stream part count changed.'
|
||||
}
|
||||
$buffer = New-Object byte[] (4MB)
|
||||
foreach ($part in $sourceParts) {
|
||||
$inputStream = [System.IO.File]::OpenRead($part)
|
||||
try {
|
||||
while (($read = $inputStream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$destinationStream.Write($buffer, 0, $read)
|
||||
$incremental.AppendData($buffer, 0, $read)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
}
|
||||
$destinationStream.Flush($true)
|
||||
$actualStreamSha = ([BitConverter]::ToString(
|
||||
$incremental.GetHashAndReset()
|
||||
)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$incremental.Dispose()
|
||||
$destinationStream.Dispose()
|
||||
}
|
||||
if ($actualStreamSha -ne $streamSha) {
|
||||
throw "Reconstructed stream SHA-256 changed: $actualStreamSha"
|
||||
}
|
||||
Move-Item -LiteralPath $inputTemp -Destination $inputPath
|
||||
}
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $image *> $null
|
||||
$imageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($imageCached) {
|
||||
Write-Host 'Using the exact cached NVIDIA DeepStream image.'
|
||||
}
|
||||
else {
|
||||
Write-Host 'Pulling exact NVIDIA DeepStream image in the interactive user session...'
|
||||
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
||||
}
|
||||
|
||||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$runRoot = Join-Path $runsRoot $runId
|
||||
$rawRoot = Join-Path $runRoot 'raw'
|
||||
$inputMount = Join-Path $runRoot 'input'
|
||||
New-Item -ItemType Directory -Force -Path $rawRoot | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'detections') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'tracks') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $inputMount | Out-Null
|
||||
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $inputMount 'right.mp4')
|
||||
$deepstreamLog = Join-Path $rawRoot 'deepstream.log'
|
||||
$trackerCopy = Join-Path $rawRoot 'tracker-config.yml'
|
||||
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
$containerCommand = @"
|
||||
set -euo pipefail
|
||||
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
||||
deepstream-app -c /workspace/package/runtime/e46e_deepstream_app.txt
|
||||
"@
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-deepstream-e46e-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=deepstream-ready-stack-e46e',
|
||||
'--label', 'com.nodedc.managed-by=mission-core-worker',
|
||||
'--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$modelsRoot,dst=/workspace/model",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
||||
)
|
||||
Write-Host 'Running full 4489-frame NVIDIA detector + NvDCF replay...'
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
||||
$deepstreamExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($deepstreamExit -ne 0) {
|
||||
throw "DeepStream replay failed with exit code $deepstreamExit"
|
||||
}
|
||||
|
||||
$overlayPath = Join-Path $rawRoot 'overlay.mp4'
|
||||
$enginePath = Join-Path $modelsRoot "$modelFile`_b1_gpu0_fp16.engine"
|
||||
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
||||
(Get-Item -LiteralPath $overlayPath).Length -eq 0) {
|
||||
throw 'DeepStream did not produce an overlay video.'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
||||
throw 'DeepStream did not produce the exact TensorRT engine.'
|
||||
}
|
||||
$detectionFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'detections') -File)
|
||||
$trackFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'tracks') -File)
|
||||
if ($detectionFiles.Count -ne [int]$profile.source.segment_count -or
|
||||
$trackFiles.Count -ne [int]$profile.source.segment_count) {
|
||||
throw "DeepStream frame coverage changed: detections=$($detectionFiles.Count), tracks=$($trackFiles.Count)"
|
||||
}
|
||||
$runtime = [ordered]@{
|
||||
schema_version = 'missioncore.e46e-deepstream-runtime/v1'
|
||||
status = 'completed'
|
||||
worker_host = $env:COMPUTERNAME
|
||||
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
||||
started_at_utc = $startedAt
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
container_image = $image
|
||||
container_image_digest = $imageDigest
|
||||
model_sha256 = Get-Sha256 $modelPath
|
||||
model_engine_sha256 = Get-Sha256 $enginePath
|
||||
deepstream_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_deepstream_app.txt')
|
||||
detector_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46e_trafficcamnet_rtdetr.txt')
|
||||
parser_library_sha256 = Get-Sha256 $parserLibraryPath
|
||||
tracker_config_sha256 = Get-Sha256 $trackerCopy
|
||||
input_stream_sha256 = Get-Sha256 $inputPath
|
||||
overlay_sha256 = Get-Sha256 $overlayPath
|
||||
frame_count = [int]$profile.source.segment_count
|
||||
deepstream_exit_code = $deepstreamExit
|
||||
}
|
||||
$runtimePath = Join-Path $rawRoot 'runtime.json'
|
||||
$runtime | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $runtimePath -Encoding UTF8
|
||||
|
||||
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$consolidatorArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46e-consolidator-$runId",
|
||||
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
||||
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
||||
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
||||
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
||||
$consolidatorImage,
|
||||
'python3', '/workspace/package/runtime/run_e46e_ready_stack.py',
|
||||
'--source-job', '/workspace/source-job',
|
||||
'--raw-root', '/workspace/raw',
|
||||
'--profile', '/workspace/package/profile.json',
|
||||
'--output-root', '/workspace/results'
|
||||
)
|
||||
Write-Host 'Freezing immutable E46E evidence...'
|
||||
Invoke-Docker $consolidatorArguments 'E46E consolidation'
|
||||
Write-Host "E46E_READY_STACK_COMPLETED run=$runId results=$resultsRoot"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$taskName = 'MissionCore-E46EReadyStack'
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$script = Join-Path $package 'runtime\Invoke-E46EReadyStack.ps1'
|
||||
if (-not (Test-Path -LiteralPath $script -PathType Leaf)) {
|
||||
throw "E46E runner is missing: $script"
|
||||
}
|
||||
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq 'Running') {
|
||||
throw "$taskName is already running."
|
||||
}
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $logsRoot | Out-Null
|
||||
$stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$logPath = Join-Path $logsRoot "e46e-ready-stack-$stamp.log"
|
||||
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = @(
|
||||
'-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', "`"$script`"",
|
||||
'-PackageRoot', "`"$package`"",
|
||||
'-SourceJobRoot', "`"$SourceJobRoot`"",
|
||||
'-RuntimeRoot', "`"$RuntimeRoot`"",
|
||||
'-LogPath', "`"$logPath`""
|
||||
) -join ' '
|
||||
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$action = New-ScheduledTaskAction -Execute $powerShell -Argument $arguments -WorkingDirectory $package
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userId -LogonType Interactive -RunLevel Limited
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||
$settings = New-ScheduledTaskSettingsSet `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable `
|
||||
-ExecutionTimeLimit ([TimeSpan]::FromHours(6))
|
||||
Register-ScheduledTask `
|
||||
-TaskName $taskName `
|
||||
-Action $action `
|
||||
-Principal $principal `
|
||||
-Trigger $trigger `
|
||||
-Settings $settings `
|
||||
-Description 'One-shot Mission Core E46E stock NVIDIA RT-DETR plus NvDCF recorded RIGHT replay.' `
|
||||
-Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Write-Host "E46E_TASK_STARTED task=$taskName log=$logPath"
|
||||
@@ -0,0 +1,329 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46f',
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label is missing: $Path"
|
||||
}
|
||||
$actual = Get-Sha256 $Path
|
||||
if ($actual -ne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
||||
$profilePath = Join-Path $package 'profile.json'
|
||||
$manifestPath = Join-Path $package 'manifest.json'
|
||||
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw 'E46F package is incomplete.'
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if ($manifest.schema_version -ne 'missioncore.e46f-worker-package/v1' -or
|
||||
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
||||
throw 'E46F package identity is invalid.'
|
||||
}
|
||||
$expectedPaths = @($manifest.identity.artifact_paths)
|
||||
foreach ($artifact in @($manifest.artifacts)) {
|
||||
if ($expectedPaths -notcontains [string]$artifact.path) {
|
||||
throw "Unexpected E46F package artifact: $($artifact.path)"
|
||||
}
|
||||
$artifactPath = Join-Path $package ([string]$artifact.path)
|
||||
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
||||
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
||||
throw "Package artifact length changed: $($artifact.path)"
|
||||
}
|
||||
}
|
||||
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
||||
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
||||
})
|
||||
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
||||
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
||||
throw 'E46F package file set changed.'
|
||||
}
|
||||
|
||||
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
||||
if ($profile.schema_version -ne 'missioncore.e46f-dashcam-bakeoff-profile/v1' -or
|
||||
$profile.comparison_contract.controlled_change -ne 'detector-only') {
|
||||
throw 'E46F profile is incompatible.'
|
||||
}
|
||||
$image = [string]$profile.runtime.container_image
|
||||
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
||||
if (-not $imageDigestMatch.Success) {
|
||||
throw 'E46F runtime image must be pinned by a full SHA-256 digest.'
|
||||
}
|
||||
$imageDigest = $imageDigestMatch.Groups[1].Value
|
||||
$modelSha = [string]$profile.detector.model_sha256
|
||||
$streamSha = [string]$profile.source.stream_sha256
|
||||
$modelFile = [string]$profile.detector.model_file
|
||||
$modelUrl = [string]$profile.detector.model_url
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
$modelsRoot = Join-Path $RuntimeRoot "models\dashcamnet\$([string]$profile.detector.version)"
|
||||
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
$resultsRoot = Join-Path $RuntimeRoot 'dashcam-bakeoff-results'
|
||||
foreach ($path in @($logsRoot, $modelsRoot, $inputsRoot, $runsRoot, $resultsRoot)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
Write-Host "E46F package: $($manifest.package_id)"
|
||||
Write-Host "E46F source: $sourceJob"
|
||||
Write-Host "E46F image: $image"
|
||||
|
||||
$modelPath = Join-Path $modelsRoot $modelFile
|
||||
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
||||
Assert-Sha256 $modelPath $modelSha 'DashCamNet model'
|
||||
}
|
||||
else {
|
||||
$modelTemp = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
||||
Write-Host 'Downloading exact NVIDIA DashCamNet model...'
|
||||
& curl.exe --fail --location --retry 3 --output $modelTemp $modelUrl
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "NVIDIA model download failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
Assert-Sha256 $modelTemp $modelSha 'downloaded DashCamNet model'
|
||||
Move-Item -LiteralPath $modelTemp -Destination $modelPath
|
||||
}
|
||||
|
||||
$jobPath = Join-Path $sourceJob 'job.json'
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if ($job.job_id -ne $profile.source.job_id -or
|
||||
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
||||
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
||||
throw 'Exact E46F source job binding changed.'
|
||||
}
|
||||
$cameraRoot = Join-Path $sourceJob 'input\camera\sensor.camera.right\epoch-1'
|
||||
$indexPath = Join-Path $cameraRoot 'index.jsonl'
|
||||
$summaryPath = Join-Path $cameraRoot 'summary.json'
|
||||
Assert-Sha256 $indexPath ([string]$profile.source.archive_index_sha256) 'source index'
|
||||
Assert-Sha256 $summaryPath ([string]$profile.source.archive_summary_sha256) 'source summary'
|
||||
$summary = Get-Content -LiteralPath $summaryPath -Raw | ConvertFrom-Json
|
||||
if ($summary.stream_sha256 -ne $streamSha -or
|
||||
[int]$summary.segment_count -ne [int]$profile.source.segment_count) {
|
||||
throw 'Source stream identity changed.'
|
||||
}
|
||||
|
||||
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
||||
$e46eInputPath = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4"
|
||||
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
||||
Assert-Sha256 $inputPath $streamSha 'E46F reconstructed RIGHT stream'
|
||||
}
|
||||
elseif (Test-Path -LiteralPath $e46eInputPath -PathType Leaf) {
|
||||
Assert-Sha256 $e46eInputPath $streamSha 'E46E controlled RIGHT stream'
|
||||
Copy-Item -LiteralPath $e46eInputPath -Destination $inputPath
|
||||
Assert-Sha256 $inputPath $streamSha 'E46F copied RIGHT stream'
|
||||
}
|
||||
else {
|
||||
$inputTemp = "$inputPath.$([Guid]::NewGuid().ToString('N')).tmp"
|
||||
$destinationStream = [System.IO.File]::Open(
|
||||
$inputTemp,
|
||||
[System.IO.FileMode]::CreateNew,
|
||||
[System.IO.FileAccess]::Write,
|
||||
[System.IO.FileShare]::None
|
||||
)
|
||||
$incremental = [System.Security.Cryptography.IncrementalHash]::CreateHash(
|
||||
[System.Security.Cryptography.HashAlgorithmName]::SHA256
|
||||
)
|
||||
try {
|
||||
$sourceParts = [System.Collections.Generic.List[string]]::new()
|
||||
$sourceParts.Add((Join-Path $cameraRoot 'init.mp4'))
|
||||
foreach ($line in [System.IO.File]::ReadLines($indexPath)) {
|
||||
$row = $line | ConvertFrom-Json
|
||||
$sourceParts.Add((Join-Path $cameraRoot ([string]$row.path)))
|
||||
}
|
||||
if ($sourceParts.Count -ne ([int]$profile.source.segment_count + 1)) {
|
||||
throw 'Source stream part count changed.'
|
||||
}
|
||||
$buffer = New-Object byte[] (4MB)
|
||||
foreach ($part in $sourceParts) {
|
||||
$inputStream = [System.IO.File]::OpenRead($part)
|
||||
try {
|
||||
while (($read = $inputStream.Read($buffer, 0, $buffer.Length)) -gt 0) {
|
||||
$destinationStream.Write($buffer, 0, $read)
|
||||
$incremental.AppendData($buffer, 0, $read)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$inputStream.Dispose()
|
||||
}
|
||||
}
|
||||
$destinationStream.Flush($true)
|
||||
$actualStreamSha = ([BitConverter]::ToString(
|
||||
$incremental.GetHashAndReset()
|
||||
)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
finally {
|
||||
$incremental.Dispose()
|
||||
$destinationStream.Dispose()
|
||||
}
|
||||
if ($actualStreamSha -ne $streamSha) {
|
||||
throw "Reconstructed stream SHA-256 changed: $actualStreamSha"
|
||||
}
|
||||
Move-Item -LiteralPath $inputTemp -Destination $inputPath
|
||||
}
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $image *> $null
|
||||
$imageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($imageCached) {
|
||||
Write-Host 'Using the exact cached NVIDIA DeepStream image.'
|
||||
}
|
||||
else {
|
||||
Write-Host 'Pulling exact NVIDIA DeepStream image in the interactive user session...'
|
||||
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
||||
}
|
||||
|
||||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$runRoot = Join-Path $runsRoot $runId
|
||||
$rawRoot = Join-Path $runRoot 'raw'
|
||||
$inputMount = Join-Path $runRoot 'input'
|
||||
New-Item -ItemType Directory -Force -Path $rawRoot | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'detections') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Join-Path $rawRoot 'tracks') | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $inputMount | Out-Null
|
||||
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $inputMount 'right.mp4')
|
||||
$deepstreamLog = Join-Path $rawRoot 'deepstream.log'
|
||||
$trackerCopy = Join-Path $rawRoot 'tracker-config.yml'
|
||||
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
$containerCommand = @"
|
||||
set -euo pipefail
|
||||
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
||||
deepstream-app -c /workspace/package/runtime/e46f_deepstream_app.txt
|
||||
"@
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-deepstream-e46f-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=deepstream-dashcam-bakeoff-e46f',
|
||||
'--label', 'com.nodedc.managed-by=mission-core-worker',
|
||||
'--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$modelsRoot,dst=/workspace/model",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
||||
)
|
||||
Write-Host 'Running full 4489-frame NVIDIA DashCamNet + NvDCF bake-off...'
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
||||
$deepstreamExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($deepstreamExit -ne 0) {
|
||||
throw "DeepStream replay failed with exit code $deepstreamExit"
|
||||
}
|
||||
$invalidOutputBinding = Select-String `
|
||||
-LiteralPath $deepstreamLog `
|
||||
-Pattern 'Could not find output layer','Given invalid tensor name' `
|
||||
-SimpleMatch `
|
||||
-Quiet
|
||||
if ($invalidOutputBinding) {
|
||||
throw 'DeepStream accepted the process but rejected the configured detector output bindings.'
|
||||
}
|
||||
|
||||
$overlayPath = Join-Path $rawRoot 'overlay.mp4'
|
||||
$enginePath = Join-Path $modelsRoot "$modelFile`_b1_gpu0_fp16.engine"
|
||||
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
||||
(Get-Item -LiteralPath $overlayPath).Length -eq 0) {
|
||||
throw 'DeepStream did not produce an overlay video.'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
||||
throw 'DeepStream did not produce the exact TensorRT engine.'
|
||||
}
|
||||
$detectionFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'detections') -File)
|
||||
$trackFiles = @(Get-ChildItem -LiteralPath (Join-Path $rawRoot 'tracks') -File)
|
||||
if ($detectionFiles.Count -ne [int]$profile.source.segment_count -or
|
||||
$trackFiles.Count -ne [int]$profile.source.segment_count) {
|
||||
throw "DeepStream frame coverage changed: detections=$($detectionFiles.Count), tracks=$($trackFiles.Count)"
|
||||
}
|
||||
$runtime = [ordered]@{
|
||||
schema_version = 'missioncore.e46f-dashcam-deepstream-runtime/v1'
|
||||
status = 'completed'
|
||||
worker_host = $env:COMPUTERNAME
|
||||
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
||||
started_at_utc = $startedAt
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
container_image = $image
|
||||
container_image_digest = $imageDigest
|
||||
model_sha256 = Get-Sha256 $modelPath
|
||||
model_engine_sha256 = Get-Sha256 $enginePath
|
||||
deepstream_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46f_deepstream_app.txt')
|
||||
detector_config_sha256 = Get-Sha256 (Join-Path $package 'runtime\e46f_dashcamnet_detectnet.txt')
|
||||
tracker_config_sha256 = Get-Sha256 $trackerCopy
|
||||
input_stream_sha256 = Get-Sha256 $inputPath
|
||||
overlay_sha256 = Get-Sha256 $overlayPath
|
||||
frame_count = [int]$profile.source.segment_count
|
||||
deepstream_exit_code = $deepstreamExit
|
||||
}
|
||||
$runtimePath = Join-Path $rawRoot 'runtime.json'
|
||||
$runtime | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $runtimePath -Encoding UTF8
|
||||
|
||||
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$consolidatorArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46f-consolidator-$runId",
|
||||
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
||||
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
||||
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
||||
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
||||
$consolidatorImage,
|
||||
'python3', '/workspace/package/runtime/run_e46f_dashcam_bakeoff.py',
|
||||
'--source-job', '/workspace/source-job',
|
||||
'--raw-root', '/workspace/raw',
|
||||
'--profile', '/workspace/package/profile.json',
|
||||
'--output-root', '/workspace/results'
|
||||
)
|
||||
Write-Host 'Freezing immutable E46F evidence...'
|
||||
Invoke-Docker $consolidatorArguments 'E46F consolidation'
|
||||
Write-Host "E46F_DASHCAM_BAKEOFF_COMPLETED run=$runId results=$resultsRoot"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46f'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$taskName = 'MissionCore-E46FDashCamBakeoff'
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$script = Join-Path $package 'runtime\Invoke-E46FDashCamBakeoff.ps1'
|
||||
if (-not (Test-Path -LiteralPath $script -PathType Leaf)) {
|
||||
throw "E46F runner is missing: $script"
|
||||
}
|
||||
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq 'Running') {
|
||||
throw "$taskName is already running."
|
||||
}
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $logsRoot | Out-Null
|
||||
$stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$logPath = Join-Path $logsRoot "e46f-dashcam-bakeoff-$stamp.log"
|
||||
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = @(
|
||||
'-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', "`"$script`"",
|
||||
'-PackageRoot', "`"$package`"",
|
||||
'-SourceJobRoot', "`"$SourceJobRoot`"",
|
||||
'-RuntimeRoot', "`"$RuntimeRoot`"",
|
||||
'-LogPath', "`"$logPath`""
|
||||
) -join ' '
|
||||
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$action = New-ScheduledTaskAction -Execute $powerShell -Argument $arguments -WorkingDirectory $package
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userId -LogonType Interactive -RunLevel Limited
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||
$settings = New-ScheduledTaskSettingsSet `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable `
|
||||
-ExecutionTimeLimit ([TimeSpan]::FromHours(6))
|
||||
Register-ScheduledTask `
|
||||
-TaskName $taskName `
|
||||
-Action $action `
|
||||
-Principal $principal `
|
||||
-Trigger $trigger `
|
||||
-Settings $settings `
|
||||
-Description 'One-shot Mission Core E46F stock NVIDIA DashCamNet plus NvDCF detector-only bake-off.' `
|
||||
-Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Write-Host "E46F_TASK_STARTED task=$taskName log=$logPath"
|
||||
@@ -0,0 +1,440 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46g',
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label is missing: $Path"
|
||||
}
|
||||
$actual = Get-Sha256 $Path
|
||||
if ($actual -ne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-VideoFrameCount([string]$Path) {
|
||||
$probe = & ffprobe -v error -select_streams v:0 -count_frames `
|
||||
-show_entries stream=nb_read_frames -of json $Path | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ffprobe failed: $Path"
|
||||
}
|
||||
return [int]@($probe.streams)[0].nb_read_frames
|
||||
}
|
||||
|
||||
function Ensure-Model(
|
||||
[pscustomobject]$Candidate,
|
||||
[string]$ModelRoot,
|
||||
[string]$Label
|
||||
) {
|
||||
New-Item -ItemType Directory -Force -Path $ModelRoot | Out-Null
|
||||
$modelPath = Join-Path $ModelRoot ([string]$Candidate.model_file)
|
||||
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
||||
Assert-Sha256 $modelPath ([string]$Candidate.model_sha256) $Label
|
||||
return $modelPath
|
||||
}
|
||||
$temporary = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
||||
& curl.exe --fail --location --retry 3 --output $temporary ([string]$Candidate.model_url)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label download failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
Assert-Sha256 $temporary ([string]$Candidate.model_sha256) "downloaded $Label"
|
||||
Move-Item -LiteralPath $temporary -Destination $modelPath
|
||||
return $modelPath
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
||||
$profilePath = Join-Path $package 'profile.json'
|
||||
$manifestPath = Join-Path $package 'manifest.json'
|
||||
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw 'E46G package is incomplete.'
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if ($manifest.schema_version -ne 'missioncore.e46g-worker-package/v1' -or
|
||||
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
||||
throw 'E46G package identity is invalid.'
|
||||
}
|
||||
$expectedPaths = @($manifest.identity.artifact_paths)
|
||||
foreach ($artifact in @($manifest.artifacts)) {
|
||||
if ($expectedPaths -notcontains [string]$artifact.path) {
|
||||
throw "Unexpected E46G package artifact: $($artifact.path)"
|
||||
}
|
||||
$artifactPath = Join-Path $package ([string]$artifact.path)
|
||||
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
||||
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
||||
throw "Package artifact length changed: $($artifact.path)"
|
||||
}
|
||||
}
|
||||
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
||||
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
||||
})
|
||||
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
||||
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
||||
throw 'E46G package file set changed.'
|
||||
}
|
||||
|
||||
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
||||
if ($profile.schema_version -ne 'missioncore.e46g-rectified-detector-bakeoff-profile/v1' -or
|
||||
$profile.source.camera_source_id -ne 'sensor.camera.right' -or
|
||||
$profile.rectification.provider -ne 'NVIDIA Gst-nvdewarper') {
|
||||
throw 'E46G profile is incompatible.'
|
||||
}
|
||||
$image = [string]$profile.runtime.container_image
|
||||
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
||||
if (-not $imageDigestMatch.Success) {
|
||||
throw 'E46G runtime image must be pinned by a full SHA-256 digest.'
|
||||
}
|
||||
$imageDigest = $imageDigestMatch.Groups[1].Value
|
||||
$streamSha = [string]$profile.source.stream_sha256
|
||||
$sampleFrameCount = [int]$profile.selection.frame_count
|
||||
$firstSourceFrame = [int]$profile.selection.first_source_frame_index
|
||||
$lastSourceFrame = [int]$profile.selection.last_source_frame_index
|
||||
$expectedFullFrameCount = [int]$profile.rectification.expected_full_frame_count
|
||||
$retainedSourceRange = @($profile.rectification.retained_source_frame_index_range)
|
||||
if ($retainedSourceRange.Count -ne 2 -or
|
||||
$firstSourceFrame -lt [int]$retainedSourceRange[0] -or
|
||||
$lastSourceFrame -gt [int]$retainedSourceRange[1]) {
|
||||
throw 'E46G selection is outside the admitted NVIDIA-decoded source prefix.'
|
||||
}
|
||||
$parserPath = Join-Path $package "runtime\$([string]$profile.trafficcamnet_parser.library_file)"
|
||||
Assert-Sha256 $parserPath ([string]$profile.trafficcamnet_parser.library_sha256) `
|
||||
'official NVIDIA DeepStream TAO parser'
|
||||
|
||||
if (-not (Get-Command ffmpeg -ErrorAction SilentlyContinue) -or
|
||||
-not (Get-Command ffprobe -ErrorAction SilentlyContinue)) {
|
||||
throw 'E46G requires the existing Worker ffmpeg/ffprobe installation.'
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
||||
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
$resultsRoot = Join-Path $RuntimeRoot 'rectified-detector-bakeoff-results'
|
||||
foreach ($path in @($inputsRoot, $runsRoot, $resultsRoot)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
$jobPath = Join-Path $sourceJob 'job.json'
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if ($job.job_id -ne $profile.source.job_id -or
|
||||
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
||||
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
||||
throw 'Exact E46G source job binding changed.'
|
||||
}
|
||||
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
||||
$e46eInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4"
|
||||
$e46fInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46f\inputs\right-$streamSha.mp4"
|
||||
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
||||
Assert-Sha256 $inputPath $streamSha 'E46G controlled RIGHT stream'
|
||||
}
|
||||
elseif (Test-Path -LiteralPath $e46eInput -PathType Leaf) {
|
||||
Assert-Sha256 $e46eInput $streamSha 'E46E controlled RIGHT stream'
|
||||
Copy-Item -LiteralPath $e46eInput -Destination $inputPath
|
||||
}
|
||||
elseif (Test-Path -LiteralPath $e46fInput -PathType Leaf) {
|
||||
Assert-Sha256 $e46fInput $streamSha 'E46F controlled RIGHT stream'
|
||||
Copy-Item -LiteralPath $e46fInput -Destination $inputPath
|
||||
}
|
||||
else {
|
||||
& (Join-Path $package 'runtime\Prepare-RectifiedCameraReplay.ps1') `
|
||||
-JobRoot $sourceJob -OutputPath $inputPath
|
||||
}
|
||||
Assert-Sha256 $inputPath $streamSha 'E46G reconstructed RIGHT stream'
|
||||
|
||||
$trafficModelRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0'
|
||||
$dashModelRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46f\models\dashcamnet\pruned_onnx_v1.0.4'
|
||||
$trafficModel = Ensure-Model $profile.candidates.trafficcamnet $trafficModelRoot 'TrafficCamNet model'
|
||||
$dashModel = Ensure-Model $profile.candidates.dashcamnet $dashModelRoot 'DashCamNet model'
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $image *> $null
|
||||
$imageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if (-not $imageCached) {
|
||||
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
||||
}
|
||||
|
||||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$runRoot = Join-Path $runsRoot $runId
|
||||
$rawRoot = Join-Path $runRoot 'raw'
|
||||
$sourceInputMount = Join-Path $runRoot 'source-input'
|
||||
$geometryRoot = Join-Path $rawRoot 'geometry'
|
||||
$samplesRoot = Join-Path $rawRoot 'samples'
|
||||
$comparisonRoot = Join-Path $rawRoot 'comparison'
|
||||
foreach ($path in @(
|
||||
$rawRoot,
|
||||
$sourceInputMount,
|
||||
$geometryRoot,
|
||||
$samplesRoot,
|
||||
$comparisonRoot
|
||||
)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $sourceInputMount 'right.mp4')
|
||||
$workerLog = Join-Path $rawRoot 'worker.log'
|
||||
"E46G run $runId`nsource=$streamSha`nselection=$firstSourceFrame..$lastSourceFrame" |
|
||||
Set-Content -LiteralPath $workerLog -Encoding UTF8
|
||||
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
$geometryRuntime = [ordered]@{}
|
||||
foreach ($view in @('left', 'front', 'right')) {
|
||||
$viewProfile = $profile.rectification.views.$view
|
||||
$configName = [string]$viewProfile.config_file
|
||||
$configPath = Join-Path $package "runtime\$configName"
|
||||
Assert-Sha256 $configPath ([string]$viewProfile.config_sha256) "$view dewarper config"
|
||||
$outputPath = Join-Path $geometryRoot "$view.mp4"
|
||||
$containerCommand = @"
|
||||
set -euo pipefail
|
||||
gst-launch-1.0 -e filesrc location=/workspace/input/right.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdewarper config-file=/workspace/package/runtime/$configName source-id=0 num-batch-buffers=1 ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=NV12' ! nvv4l2h264enc bitrate=6000000 ! h264parse ! qtmux ! filesink location=/workspace/output/$view.mp4
|
||||
"@
|
||||
$dewarperLog = Join-Path $geometryRoot "$view.log"
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46g-dewarper-$view-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=nvdewarper-e46g',
|
||||
'--mount', "type=bind,src=$sourceInputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$geometryRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
||||
)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $dewarperLog
|
||||
$dewarperExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($dewarperExit -ne 0 -or -not (Test-Path -LiteralPath $outputPath -PathType Leaf)) {
|
||||
throw "NVIDIA nvdewarper failed for $view with exit code $dewarperExit"
|
||||
}
|
||||
if ((Get-VideoFrameCount $outputPath) -ne $expectedFullFrameCount) {
|
||||
throw "Full rectified $view frame coverage changed."
|
||||
}
|
||||
|
||||
$samplePath = Join-Path $samplesRoot "$view.mp4"
|
||||
$filter = "select='between(n\,$firstSourceFrame\,$lastSourceFrame)',setpts=N/(10*TB)"
|
||||
& ffmpeg -hide_banner -loglevel error -y -i $outputPath -vf $filter -an `
|
||||
-c:v libx264 -preset fast -crf 18 -pix_fmt yuv420p -r 10 $samplePath
|
||||
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $samplePath) -ne $sampleFrameCount) {
|
||||
throw "Exact E46G sample extraction failed for $view."
|
||||
}
|
||||
$geometryRuntime[$view] = [ordered]@{
|
||||
dewarper_config_sha256 = Get-Sha256 $configPath
|
||||
dewarper_log_sha256 = Get-Sha256 $dewarperLog
|
||||
full_rectified_video_path = "geometry/$view.mp4"
|
||||
full_rectified_video_sha256 = Get-Sha256 $outputPath
|
||||
sample_video_path = "samples/$view.mp4"
|
||||
sample_video_sha256 = Get-Sha256 $samplePath
|
||||
full_frame_count = $expectedFullFrameCount
|
||||
retained_source_frame_index_range = @(
|
||||
[int]$retainedSourceRange[0],
|
||||
[int]$retainedSourceRange[1]
|
||||
)
|
||||
excluded_source_tail_frame_count = [int]$profile.rectification.excluded_source_tail_frame_count
|
||||
sample_frame_count = $sampleFrameCount
|
||||
}
|
||||
}
|
||||
|
||||
$candidateRuntime = [ordered]@{}
|
||||
$candidateDefinitions = @(
|
||||
[pscustomobject]@{
|
||||
Name = 'trafficcamnet'
|
||||
ModelRoot = $trafficModelRoot
|
||||
ModelPath = $trafficModel
|
||||
},
|
||||
[pscustomobject]@{
|
||||
Name = 'dashcamnet'
|
||||
ModelRoot = $dashModelRoot
|
||||
ModelPath = $dashModel
|
||||
}
|
||||
)
|
||||
foreach ($definition in $candidateDefinitions) {
|
||||
$candidate = [string]$definition.Name
|
||||
$candidateProfile = $profile.candidates.$candidate
|
||||
$appConfigPath = Join-Path $package "runtime\$([string]$candidateProfile.deepstream_app_config)"
|
||||
$detectorConfigPath = Join-Path $package "runtime\$([string]$candidateProfile.detector_config)"
|
||||
Assert-Sha256 $appConfigPath ([string]$candidateProfile.deepstream_app_config_sha256) `
|
||||
"$candidate DeepStream app config"
|
||||
Assert-Sha256 $detectorConfigPath ([string]$candidateProfile.detector_config_sha256) `
|
||||
"$candidate detector config"
|
||||
$runs = [ordered]@{}
|
||||
foreach ($view in @('left', 'front', 'right')) {
|
||||
$viewRoot = Join-Path $rawRoot "runs\$candidate\$view"
|
||||
$inputMount = Join-Path $viewRoot 'input'
|
||||
foreach ($path in @(
|
||||
$viewRoot,
|
||||
$inputMount,
|
||||
(Join-Path $viewRoot 'detections'),
|
||||
(Join-Path $viewRoot 'tracks')
|
||||
)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
Copy-Item -LiteralPath (Join-Path $samplesRoot "$view.mp4") `
|
||||
-Destination (Join-Path $inputMount 'view.mp4')
|
||||
$deepstreamLog = Join-Path $viewRoot 'deepstream.log'
|
||||
$containerCommand = @"
|
||||
set -euo pipefail
|
||||
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
||||
deepstream-app -c /workspace/package/runtime/$([string]$candidateProfile.deepstream_app_config)
|
||||
"@
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46g-$candidate-$view-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', "com.nodedc.role=e46g-$candidate-$view",
|
||||
'--mount', "type=bind,src=$inputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$($definition.ModelRoot),dst=/workspace/model",
|
||||
'--mount', "type=bind,src=$viewRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $containerCommand
|
||||
)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dockerArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
||||
$deepstreamExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($deepstreamExit -ne 0) {
|
||||
throw "DeepStream failed for $candidate/$view with exit code $deepstreamExit"
|
||||
}
|
||||
$overlayPath = Join-Path $viewRoot 'overlay.mp4'
|
||||
$trackerPath = Join-Path $viewRoot 'tracker-config.yml'
|
||||
$detections = @(Get-ChildItem -LiteralPath (Join-Path $viewRoot 'detections') -File)
|
||||
$tracks = @(Get-ChildItem -LiteralPath (Join-Path $viewRoot 'tracks') -File)
|
||||
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
||||
$detections.Count -ne $sampleFrameCount -or $tracks.Count -ne $sampleFrameCount) {
|
||||
throw "DeepStream output coverage changed for $candidate/$view."
|
||||
}
|
||||
$enginePath = "$($definition.ModelPath)_b1_gpu0_fp16.engine"
|
||||
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
||||
throw "TensorRT engine is missing for $candidate."
|
||||
}
|
||||
$runs[$view] = [ordered]@{
|
||||
overlay_path = "runs/$candidate/$view/overlay.mp4"
|
||||
overlay_sha256 = Get-Sha256 $overlayPath
|
||||
deepstream_log_path = "runs/$candidate/$view/deepstream.log"
|
||||
deepstream_log_sha256 = Get-Sha256 $deepstreamLog
|
||||
tracker_config_sha256 = Get-Sha256 $trackerPath
|
||||
model_engine_sha256 = Get-Sha256 $enginePath
|
||||
frame_count = $sampleFrameCount
|
||||
deepstream_exit_code = $deepstreamExit
|
||||
}
|
||||
}
|
||||
$candidateRuntime[$candidate] = [ordered]@{
|
||||
model_sha256 = Get-Sha256 $definition.ModelPath
|
||||
deepstream_app_config_sha256 = Get-Sha256 $appConfigPath
|
||||
detector_config_sha256 = Get-Sha256 $detectorConfigPath
|
||||
parser_library_sha256 = $(if ($candidate -eq 'trafficcamnet') {
|
||||
Get-Sha256 $parserPath
|
||||
} else { $null })
|
||||
runs = $runs
|
||||
}
|
||||
}
|
||||
|
||||
$comparisonRuntime = [ordered]@{}
|
||||
foreach ($candidate in @('trafficcamnet', 'dashcamnet')) {
|
||||
$left = Join-Path $rawRoot "runs\$candidate\left\overlay.mp4"
|
||||
$front = Join-Path $rawRoot "runs\$candidate\front\overlay.mp4"
|
||||
$right = Join-Path $rawRoot "runs\$candidate\right\overlay.mp4"
|
||||
$output = Join-Path $comparisonRoot "$candidate.mp4"
|
||||
& ffmpeg -hide_banner -loglevel error -y -i $left -i $front -i $right `
|
||||
-filter_complex '[0:v][1:v][2:v]hstack=inputs=3[v]' -map '[v]' -an `
|
||||
-c:v libx264 -preset fast -crf 20 -pix_fmt yuv420p -movflags +faststart $output
|
||||
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $output) -ne $sampleFrameCount) {
|
||||
throw "E46G synchronized comparison video failed for $candidate."
|
||||
}
|
||||
$comparisonRuntime[$candidate] = [ordered]@{
|
||||
video_path = "comparison/$candidate.mp4"
|
||||
video_sha256 = Get-Sha256 $output
|
||||
frame_count = $sampleFrameCount
|
||||
view_order = @('left', 'front', 'right')
|
||||
}
|
||||
}
|
||||
|
||||
$runtime = [ordered]@{
|
||||
schema_version = 'missioncore.e46g-rectified-detector-runtime/v1'
|
||||
status = 'completed'
|
||||
worker_host = $env:COMPUTERNAME
|
||||
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
||||
started_at_utc = $startedAt
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
container_image = $image
|
||||
container_image_digest = $imageDigest
|
||||
source_stream_sha256 = Get-Sha256 $inputPath
|
||||
first_source_frame_index = $firstSourceFrame
|
||||
sample_frame_count = $sampleFrameCount
|
||||
geometry = $geometryRuntime
|
||||
candidates = $candidateRuntime
|
||||
comparison = $comparisonRuntime
|
||||
}
|
||||
$runtime | ConvertTo-Json -Depth 16 |
|
||||
Set-Content -LiteralPath (Join-Path $rawRoot 'runtime.json') -Encoding UTF8
|
||||
Add-Content -LiteralPath $workerLog -Value "completed=$(Get-Date -Format o)"
|
||||
|
||||
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$consolidatorArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46g-consolidator-$runId",
|
||||
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
||||
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
||||
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
||||
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
||||
$consolidatorImage,
|
||||
'python3', '/workspace/package/runtime/run_e46g_rectified_detector_bakeoff.py',
|
||||
'--source-job', '/workspace/source-job',
|
||||
'--raw-root', '/workspace/raw',
|
||||
'--profile', '/workspace/package/profile.json',
|
||||
'--output-root', '/workspace/results'
|
||||
)
|
||||
Invoke-Docker $consolidatorArguments 'E46G consolidation'
|
||||
Write-Host "E46G_RECTIFIED_DETECTOR_BAKEOFF_COMPLETED run=$runId results=$resultsRoot"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46g'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$taskName = 'MissionCore-E46GRectifiedDetectorBakeoff'
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$script = Join-Path $package 'runtime\Invoke-E46GRectifiedDetectorBakeoff.ps1'
|
||||
if (-not (Test-Path -LiteralPath $script -PathType Leaf)) {
|
||||
throw "E46G runner is missing: $script"
|
||||
}
|
||||
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq 'Running') {
|
||||
throw "$taskName is already running."
|
||||
}
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $logsRoot | Out-Null
|
||||
$stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$logPath = Join-Path $logsRoot "e46g-rectified-detector-bakeoff-$stamp.log"
|
||||
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = @(
|
||||
'-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', "`"$script`"",
|
||||
'-PackageRoot', "`"$package`"",
|
||||
'-SourceJobRoot', "`"$SourceJobRoot`"",
|
||||
'-RuntimeRoot', "`"$RuntimeRoot`"",
|
||||
'-LogPath', "`"$logPath`""
|
||||
) -join ' '
|
||||
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$action = New-ScheduledTaskAction -Execute $powerShell -Argument $arguments -WorkingDirectory $package
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userId -LogonType Interactive -RunLevel Limited
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||
$settings = New-ScheduledTaskSettingsSet `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable `
|
||||
-ExecutionTimeLimit ([TimeSpan]::FromHours(6))
|
||||
Register-ScheduledTask `
|
||||
-TaskName $taskName `
|
||||
-Action $action `
|
||||
-Principal $principal `
|
||||
-Trigger $trigger `
|
||||
-Settings $settings `
|
||||
-Description 'One-shot E46G factory-KB4 NVIDIA nvdewarper detector A/B.' `
|
||||
-Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Write-Host "E46G_TASK_STARTED task=$taskName log=$logPath"
|
||||
@@ -0,0 +1,369 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46h',
|
||||
|
||||
[string]$LogPath = ''
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-Sha256([string]$Path, [string]$Expected, [string]$Label) {
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "$Label is missing: $Path"
|
||||
}
|
||||
$actual = Get-Sha256 $Path
|
||||
if ($actual -ne $Expected) {
|
||||
throw "$Label SHA-256 changed: expected $Expected, got $actual"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Docker([string[]]$Arguments, [string]$Label) {
|
||||
& docker @Arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Label failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-VideoFrameCount([string]$Path) {
|
||||
$probe = & ffprobe -v error -select_streams v:0 -count_frames `
|
||||
-show_entries stream=nb_read_frames -of json $Path | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ffprobe failed: $Path"
|
||||
}
|
||||
return [int]@($probe.streams)[0].nb_read_frames
|
||||
}
|
||||
|
||||
function Ensure-Model([pscustomobject]$Detector, [string]$ModelRoot) {
|
||||
New-Item -ItemType Directory -Force -Path $ModelRoot | Out-Null
|
||||
$modelPath = Join-Path $ModelRoot ([string]$Detector.model_file)
|
||||
if (Test-Path -LiteralPath $modelPath -PathType Leaf) {
|
||||
Assert-Sha256 $modelPath ([string]$Detector.model_sha256) 'TrafficCamNet model'
|
||||
return $modelPath
|
||||
}
|
||||
$temporary = "$modelPath.$([Guid]::NewGuid().ToString('N')).download"
|
||||
& curl.exe --fail --location --retry 3 --output $temporary ([string]$Detector.model_url)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "TrafficCamNet download failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
Assert-Sha256 $temporary ([string]$Detector.model_sha256) 'downloaded TrafficCamNet model'
|
||||
Move-Item -LiteralPath $temporary -Destination $modelPath
|
||||
return $modelPath
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$sourceJob = (Resolve-Path -LiteralPath $SourceJobRoot).Path
|
||||
$profilePath = Join-Path $package 'profile.json'
|
||||
$manifestPath = Join-Path $package 'manifest.json'
|
||||
if (-not (Test-Path -LiteralPath $profilePath -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
throw 'E46H package is incomplete.'
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if ($manifest.schema_version -ne 'missioncore.e46h-worker-package/v1' -or
|
||||
$manifest.package_id -ne (Split-Path -Leaf $package)) {
|
||||
throw 'E46H package identity is invalid.'
|
||||
}
|
||||
$expectedPaths = @($manifest.identity.artifact_paths)
|
||||
foreach ($artifact in @($manifest.artifacts)) {
|
||||
if ($expectedPaths -notcontains [string]$artifact.path) {
|
||||
throw "Unexpected E46H package artifact: $($artifact.path)"
|
||||
}
|
||||
$artifactPath = Join-Path $package ([string]$artifact.path)
|
||||
Assert-Sha256 $artifactPath ([string]$artifact.sha256) "package artifact $($artifact.path)"
|
||||
if ((Get-Item -LiteralPath $artifactPath).Length -ne [int64]$artifact.byte_length) {
|
||||
throw "Package artifact length changed: $($artifact.path)"
|
||||
}
|
||||
}
|
||||
$actualPaths = @(Get-ChildItem -LiteralPath $package -Recurse -File | ForEach-Object {
|
||||
$_.FullName.Substring($package.Length + 1).Replace('\', '/')
|
||||
})
|
||||
if (@($actualPaths | Where-Object { $_ -ne 'manifest.json' -and $expectedPaths -notcontains $_ }).Count -ne 0 -or
|
||||
@($expectedPaths | Where-Object { $actualPaths -notcontains $_ }).Count -ne 0) {
|
||||
throw 'E46H package file set changed.'
|
||||
}
|
||||
|
||||
$profile = Get-Content -LiteralPath $profilePath -Raw | ConvertFrom-Json
|
||||
if ($profile.schema_version -ne 'missioncore.e46h-full-rectified-front-replay-profile/v1' -or
|
||||
$profile.source.camera_source_id -ne 'sensor.camera.right' -or
|
||||
$profile.rectification.view -ne 'front' -or
|
||||
$profile.detector.name -ne 'NVIDIA TrafficCamNet Transformer Lite') {
|
||||
throw 'E46H profile is incompatible.'
|
||||
}
|
||||
$image = [string]$profile.runtime.container_image
|
||||
$imageDigestMatch = [regex]::Match($image, '@sha256:([0-9a-f]{64})$')
|
||||
if (-not $imageDigestMatch.Success) {
|
||||
throw 'E46H runtime image must be pinned by a full SHA-256 digest.'
|
||||
}
|
||||
$imageDigest = $imageDigestMatch.Groups[1].Value
|
||||
$streamSha = [string]$profile.source.stream_sha256
|
||||
$frameCount = [int]$profile.selection.frame_count
|
||||
if ($frameCount -ne 4488 -or
|
||||
[int]$profile.selection.first_source_frame_index -ne 0 -or
|
||||
[int]$profile.selection.last_source_frame_index -ne 4487) {
|
||||
throw 'E46H retained route contract changed.'
|
||||
}
|
||||
$parserPath = Join-Path $package "runtime\$([string]$profile.parser.library_file)"
|
||||
Assert-Sha256 $parserPath ([string]$profile.parser.library_sha256) `
|
||||
'official NVIDIA DeepStream TAO parser'
|
||||
|
||||
if (-not (Get-Command ffmpeg -ErrorAction SilentlyContinue) -or
|
||||
-not (Get-Command ffprobe -ErrorAction SilentlyContinue)) {
|
||||
throw 'E46H requires the existing Worker ffmpeg/ffprobe installation.'
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $RuntimeRoot | Out-Null
|
||||
$inputsRoot = Join-Path $RuntimeRoot 'inputs'
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
$resultsRoot = Join-Path $RuntimeRoot 'full-rectified-front-results'
|
||||
foreach ($path in @($inputsRoot, $runsRoot, $resultsRoot)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
if ($LogPath) {
|
||||
$logParent = Split-Path -Parent $LogPath
|
||||
if ($logParent) { New-Item -ItemType Directory -Force -Path $logParent | Out-Null }
|
||||
Start-Transcript -LiteralPath $LogPath -Append | Out-Null
|
||||
}
|
||||
|
||||
try {
|
||||
$jobPath = Join-Path $sourceJob 'job.json'
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if ($job.job_id -ne $profile.source.job_id -or
|
||||
$job.input.archive_index_sha256 -ne $profile.source.archive_index_sha256 -or
|
||||
$job.input.archive_summary_sha256 -ne $profile.source.archive_summary_sha256) {
|
||||
throw 'Exact E46H source job binding changed.'
|
||||
}
|
||||
$inputPath = Join-Path $inputsRoot "right-$streamSha.mp4"
|
||||
$e46gInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46g\inputs\right-$streamSha.mp4"
|
||||
$e46eInput = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-$streamSha.mp4"
|
||||
if (Test-Path -LiteralPath $inputPath -PathType Leaf) {
|
||||
Assert-Sha256 $inputPath $streamSha 'E46H controlled RIGHT stream'
|
||||
}
|
||||
elseif (Test-Path -LiteralPath $e46gInput -PathType Leaf) {
|
||||
Assert-Sha256 $e46gInput $streamSha 'E46G controlled RIGHT stream'
|
||||
Copy-Item -LiteralPath $e46gInput -Destination $inputPath
|
||||
}
|
||||
elseif (Test-Path -LiteralPath $e46eInput -PathType Leaf) {
|
||||
Assert-Sha256 $e46eInput $streamSha 'E46E controlled RIGHT stream'
|
||||
Copy-Item -LiteralPath $e46eInput -Destination $inputPath
|
||||
}
|
||||
else {
|
||||
& (Join-Path $package 'runtime\Prepare-RectifiedCameraReplay.ps1') `
|
||||
-JobRoot $sourceJob -OutputPath $inputPath
|
||||
}
|
||||
Assert-Sha256 $inputPath $streamSha 'E46H reconstructed RIGHT stream'
|
||||
|
||||
$modelRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\models\trafficcamnet_transformer_lite\deployable_resnet50_v2.0'
|
||||
$modelPath = Ensure-Model $profile.detector $modelRoot
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker image inspect $image *> $null
|
||||
$imageCached = $LASTEXITCODE -eq 0
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if (-not $imageCached) {
|
||||
Invoke-Docker @('pull', $image) 'DeepStream image pull'
|
||||
}
|
||||
|
||||
$runId = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$runRoot = Join-Path $runsRoot $runId
|
||||
$rawRoot = Join-Path $runRoot 'raw'
|
||||
$sourceInputMount = Join-Path $runRoot 'source-input'
|
||||
$geometryRoot = Join-Path $rawRoot 'geometry'
|
||||
$runOutput = Join-Path $rawRoot 'run'
|
||||
$frontInputMount = Join-Path $runRoot 'front-input'
|
||||
foreach ($path in @(
|
||||
$rawRoot,
|
||||
$sourceInputMount,
|
||||
$geometryRoot,
|
||||
$runOutput,
|
||||
$frontInputMount,
|
||||
(Join-Path $runOutput 'detections'),
|
||||
(Join-Path $runOutput 'tracks')
|
||||
)) {
|
||||
New-Item -ItemType Directory -Force -Path $path | Out-Null
|
||||
}
|
||||
Copy-Item -LiteralPath $inputPath -Destination (Join-Path $sourceInputMount 'right.mp4')
|
||||
$workerLog = Join-Path $rawRoot 'worker.log'
|
||||
"E46H run $runId`nsource=$streamSha`nselection=0..4487`nview=front" |
|
||||
Set-Content -LiteralPath $workerLog -Encoding UTF8
|
||||
$startedAt = (Get-Date).ToUniversalTime().ToString('o')
|
||||
|
||||
$dewarperConfig = Join-Path $package "runtime\$([string]$profile.rectification.config_file)"
|
||||
Assert-Sha256 $dewarperConfig ([string]$profile.rectification.config_sha256) `
|
||||
'FRONT dewarper config'
|
||||
$frontPath = Join-Path $geometryRoot 'front.mp4'
|
||||
$dewarperLog = Join-Path $geometryRoot 'front.log'
|
||||
$dewarperCommand = @"
|
||||
set -euo pipefail
|
||||
gst-launch-1.0 -e filesrc location=/workspace/input/right.mp4 ! qtdemux ! h264parse ! nvv4l2decoder ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdewarper config-file=/workspace/package/runtime/$([string]$profile.rectification.config_file) source-id=0 num-batch-buffers=1 ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=NV12' ! nvv4l2h264enc bitrate=6000000 ! h264parse ! qtmux ! filesink location=/workspace/output/front.mp4
|
||||
"@
|
||||
$dewarperArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46h-dewarper-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=nvdewarper-e46h-front',
|
||||
'--mount', "type=bind,src=$sourceInputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$geometryRoot,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $dewarperCommand
|
||||
)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @dewarperArguments 2>&1 | Tee-Object -LiteralPath $dewarperLog
|
||||
$dewarperExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($dewarperExit -ne 0 -or -not (Test-Path -LiteralPath $frontPath -PathType Leaf)) {
|
||||
throw "NVIDIA nvdewarper failed with exit code $dewarperExit"
|
||||
}
|
||||
if ((Get-VideoFrameCount $frontPath) -ne $frameCount) {
|
||||
throw 'E46H FRONT frame coverage changed.'
|
||||
}
|
||||
$normalizedFront = Join-Path $geometryRoot 'front-normalized.mp4'
|
||||
& ffmpeg -hide_banner -loglevel error -y -fflags +genpts -i $frontPath `
|
||||
-vf 'setpts=N/(10*TB)' -an -c:v libx264 -preset fast -crf 18 `
|
||||
-pix_fmt yuv420p -r 10 -movflags +faststart $normalizedFront
|
||||
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $normalizedFront) -ne $frameCount) {
|
||||
throw 'E46H FRONT timestamp normalization failed.'
|
||||
}
|
||||
Move-Item -LiteralPath $normalizedFront -Destination $frontPath -Force
|
||||
Copy-Item -LiteralPath $frontPath -Destination (Join-Path $frontInputMount 'view.mp4')
|
||||
|
||||
$appConfig = Join-Path $package "runtime\$([string]$profile.detector.deepstream_app_config)"
|
||||
$detectorConfig = Join-Path $package "runtime\$([string]$profile.detector.detector_config)"
|
||||
Assert-Sha256 $appConfig ([string]$profile.detector.deepstream_app_config_sha256) `
|
||||
'TrafficCamNet DeepStream app config'
|
||||
Assert-Sha256 $detectorConfig ([string]$profile.detector.detector_config_sha256) `
|
||||
'TrafficCamNet detector config'
|
||||
$deepstreamLog = Join-Path $runOutput 'deepstream.log'
|
||||
$deepstreamCommand = @"
|
||||
set -euo pipefail
|
||||
cp /opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml /workspace/output/tracker-config.yml
|
||||
deepstream-app -c /workspace/package/runtime/$([string]$profile.detector.deepstream_app_config)
|
||||
"@
|
||||
$deepstreamArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46h-front-$runId",
|
||||
'--gpus', 'all', '--network', 'none', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--shm-size', '4g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=e46h-front-trafficcamnet',
|
||||
'--mount', "type=bind,src=$frontInputMount,dst=/workspace/input,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$modelRoot,dst=/workspace/model",
|
||||
'--mount', "type=bind,src=$runOutput,dst=/workspace/output",
|
||||
'--entrypoint', '/bin/bash', $image, '-lc', $deepstreamCommand
|
||||
)
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& docker @deepstreamArguments 2>&1 | Tee-Object -LiteralPath $deepstreamLog
|
||||
$deepstreamExit = $LASTEXITCODE
|
||||
}
|
||||
finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
if ($deepstreamExit -ne 0) {
|
||||
throw "DeepStream failed with exit code $deepstreamExit"
|
||||
}
|
||||
$overlayPath = Join-Path $runOutput 'overlay.mp4'
|
||||
$trackerPath = Join-Path $runOutput 'tracker-config.yml'
|
||||
$detections = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'detections') -File)
|
||||
$tracks = @(Get-ChildItem -LiteralPath (Join-Path $runOutput 'tracks') -File)
|
||||
if (-not (Test-Path -LiteralPath $overlayPath -PathType Leaf) -or
|
||||
$detections.Count -ne $frameCount -or $tracks.Count -ne $frameCount) {
|
||||
throw "DeepStream output coverage changed: detections=$($detections.Count), tracks=$($tracks.Count)"
|
||||
}
|
||||
$fastOverlay = Join-Path $runOutput 'overlay-faststart.mp4'
|
||||
& ffmpeg -hide_banner -loglevel error -y -i $overlayPath -c copy -movflags +faststart $fastOverlay
|
||||
if ($LASTEXITCODE -ne 0 -or (Get-VideoFrameCount $fastOverlay) -ne $frameCount) {
|
||||
throw 'E46H fast-start overlay normalization failed.'
|
||||
}
|
||||
Move-Item -LiteralPath $fastOverlay -Destination $overlayPath -Force
|
||||
$enginePath = "$modelPath`_b1_gpu0_fp16.engine"
|
||||
if (-not (Test-Path -LiteralPath $enginePath -PathType Leaf)) {
|
||||
throw 'TrafficCamNet TensorRT engine is missing.'
|
||||
}
|
||||
|
||||
$runtime = [ordered]@{
|
||||
schema_version = 'missioncore.e46h-full-rectified-front-runtime/v1'
|
||||
status = 'completed'
|
||||
worker_host = $env:COMPUTERNAME
|
||||
gpu_name = ((& nvidia-smi --query-gpu=name --format=csv,noheader | Select-Object -First 1).Trim())
|
||||
started_at_utc = $startedAt
|
||||
completed_at_utc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
container_image = $image
|
||||
container_image_digest = $imageDigest
|
||||
source_stream_sha256 = Get-Sha256 $inputPath
|
||||
frame_count = $frameCount
|
||||
retained_source_frame_index_range = @(0, 4487)
|
||||
geometry = [ordered]@{
|
||||
video_path = 'geometry/front.mp4'
|
||||
video_sha256 = Get-Sha256 $frontPath
|
||||
log_path = 'geometry/front.log'
|
||||
log_sha256 = Get-Sha256 $dewarperLog
|
||||
config_sha256 = Get-Sha256 $dewarperConfig
|
||||
frame_count = $frameCount
|
||||
dewarper_exit_code = $dewarperExit
|
||||
}
|
||||
run = [ordered]@{
|
||||
overlay_path = 'run/overlay.mp4'
|
||||
overlay_sha256 = Get-Sha256 $overlayPath
|
||||
deepstream_log_path = 'run/deepstream.log'
|
||||
deepstream_log_sha256 = Get-Sha256 $deepstreamLog
|
||||
model_sha256 = Get-Sha256 $modelPath
|
||||
model_engine_sha256 = Get-Sha256 $enginePath
|
||||
parser_library_sha256 = Get-Sha256 $parserPath
|
||||
deepstream_app_config_sha256 = Get-Sha256 $appConfig
|
||||
detector_config_sha256 = Get-Sha256 $detectorConfig
|
||||
tracker_config_sha256 = Get-Sha256 $trackerPath
|
||||
frame_count = $frameCount
|
||||
deepstream_exit_code = $deepstreamExit
|
||||
}
|
||||
}
|
||||
$runtime | ConvertTo-Json -Depth 12 |
|
||||
Set-Content -LiteralPath (Join-Path $rawRoot 'runtime.json') -Encoding UTF8
|
||||
Add-Content -LiteralPath $workerLog -Value "completed=$(Get-Date -Format o)"
|
||||
|
||||
$consolidatorImage = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$consolidatorArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46h-consolidator-$runId",
|
||||
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges', '--tmpfs', '/tmp:rw,noexec,nosuid,size=64m',
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$sourceJob,dst=/workspace/source-job,readonly",
|
||||
'--mount', "type=bind,src=$rawRoot,dst=/workspace/raw,readonly",
|
||||
'--mount', "type=bind,src=$resultsRoot,dst=/workspace/results",
|
||||
'-e', 'PYTHONPATH=/workspace/package/runtime',
|
||||
'-e', 'PYTHONDONTWRITEBYTECODE=1',
|
||||
$consolidatorImage,
|
||||
'python3', '/workspace/package/runtime/run_e46h_full_rectified_front_replay.py',
|
||||
'--source-job', '/workspace/source-job',
|
||||
'--raw-root', '/workspace/raw',
|
||||
'--profile', '/workspace/package/profile.json',
|
||||
'--output-root', '/workspace/results'
|
||||
)
|
||||
Invoke-Docker $consolidatorArguments 'E46H consolidation'
|
||||
Write-Host "E46H_FULL_RECTIFIED_FRONT_COMPLETED run=$runId results=$resultsRoot"
|
||||
}
|
||||
finally {
|
||||
if ($LogPath) { Stop-Transcript | Out-Null }
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$SourceJobRoot = 'D:\NDC_MISSIONCORE\runtime\jobs\recorded-camera-602ac89026ed12978619801d',
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46h'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$taskName = 'MissionCore-E46HFullRectifiedFrontReplay'
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$script = Join-Path $package 'runtime\Invoke-E46HFullRectifiedFrontReplay.ps1'
|
||||
if (-not (Test-Path -LiteralPath $script -PathType Leaf)) {
|
||||
throw "E46H runner is missing: $script"
|
||||
}
|
||||
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
||||
if ($existing -and $existing.State -eq 'Running') {
|
||||
throw "$taskName is already running."
|
||||
}
|
||||
$logsRoot = Join-Path $RuntimeRoot 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $logsRoot | Out-Null
|
||||
$stamp = (Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ')
|
||||
$logPath = Join-Path $logsRoot "e46h-full-rectified-front-$stamp.log"
|
||||
$powerShell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = @(
|
||||
'-NoLogo', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass',
|
||||
'-File', "`"$script`"",
|
||||
'-PackageRoot', "`"$package`"",
|
||||
'-SourceJobRoot', "`"$SourceJobRoot`"",
|
||||
'-RuntimeRoot', "`"$RuntimeRoot`"",
|
||||
'-LogPath', "`"$logPath`""
|
||||
) -join ' '
|
||||
$userId = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
|
||||
$action = New-ScheduledTaskAction -Execute $powerShell -Argument $arguments -WorkingDirectory $package
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $userId -LogonType Interactive -RunLevel Limited
|
||||
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
||||
$settings = New-ScheduledTaskSettingsSet `
|
||||
-AllowStartIfOnBatteries `
|
||||
-DontStopIfGoingOnBatteries `
|
||||
-StartWhenAvailable `
|
||||
-ExecutionTimeLimit ([TimeSpan]::FromHours(6))
|
||||
Register-ScheduledTask `
|
||||
-TaskName $taskName `
|
||||
-Action $action `
|
||||
-Principal $principal `
|
||||
-Trigger $trigger `
|
||||
-Settings $settings `
|
||||
-Description 'One-shot E46H full retained FRONT TrafficCamNet + NvDCF replay.' `
|
||||
-Force | Out-Null
|
||||
Start-ScheduledTask -TaskName $taskName
|
||||
Write-Host "E46H_TASK_STARTED task=$taskName log=$logPath"
|
||||
@@ -0,0 +1,118 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
|
||||
[string]$RuntimeRoot = 'D:\NDC_MISSIONCORE\runtime\experiments\e46j',
|
||||
|
||||
[string]$SourceVideo = 'D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4',
|
||||
|
||||
[string]$ValidFovMask = 'D:\NDC_MISSIONCORE\runtime\inputs\e2\valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2\mask.png',
|
||||
|
||||
[int]$MaxFrames = 0,
|
||||
|
||||
[switch]$NoOverlay,
|
||||
|
||||
[string]$RunPrefix = 'full'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
function Get-Sha256([string]$Path) {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
$package = (Resolve-Path -LiteralPath $PackageRoot).Path
|
||||
$source = (Resolve-Path -LiteralPath $SourceVideo).Path
|
||||
$mask = (Resolve-Path -LiteralPath $ValidFovMask).Path
|
||||
$profile = Join-Path $package 'e46j_raw_fisheye_yolox_profile.json'
|
||||
$runner = Join-Path $package 'run_e46j_raw_fisheye_yolox.py'
|
||||
if (-not (Test-Path -LiteralPath $profile -PathType Leaf) -or
|
||||
-not (Test-Path -LiteralPath $runner -PathType Leaf)) {
|
||||
throw 'E46J package is incomplete.'
|
||||
}
|
||||
if ((Get-Sha256 $source) -ne 'cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8') {
|
||||
throw 'E46J source stream identity changed.'
|
||||
}
|
||||
|
||||
$runsRoot = Join-Path $RuntimeRoot 'runs'
|
||||
New-Item -ItemType Directory -Force -Path $runsRoot | Out-Null
|
||||
$runId = "$RunPrefix-$((Get-Date).ToUniversalTime().ToString('yyyyMMddTHHmmssfffZ'))"
|
||||
$image = 'nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794'
|
||||
$command = "python3 /workspace/package/run_e46j_raw_fisheye_yolox.py --input /workspace/input/right.mp4 --mask /workspace/valid-fov/mask.png --profile /workspace/package/e46j_raw_fisheye_yolox_profile.json --triton-url http://127.0.0.1:8000 --output /workspace/runs/$runId"
|
||||
if ($MaxFrames -gt 0) {
|
||||
$command += " --max-frames $MaxFrames"
|
||||
}
|
||||
if (-not $NoOverlay) {
|
||||
$command += ' --overlay'
|
||||
}
|
||||
|
||||
$dockerArguments = @(
|
||||
'run', '--rm', '--name', "ndc-mission-core-e46j-$runId",
|
||||
'--network', 'container:ndc-mission-core-triton',
|
||||
'--gpus', 'all',
|
||||
'--cap-drop', 'ALL',
|
||||
'--security-opt', 'no-new-privileges',
|
||||
'--shm-size', '1g',
|
||||
'--label', 'com.nodedc.product=mission-core',
|
||||
'--label', 'com.nodedc.stack=perception',
|
||||
'--label', 'com.nodedc.role=e46j-raw-fisheye-realtime-gate',
|
||||
'--label', 'com.nodedc.managed-by=mission-core-worker',
|
||||
'--mount', "type=bind,src=$source,dst=/workspace/input/right.mp4,readonly",
|
||||
'--mount', "type=bind,src=$mask,dst=/workspace/valid-fov/mask.png,readonly",
|
||||
'--mount', "type=bind,src=$package,dst=/workspace/package,readonly",
|
||||
'--mount', "type=bind,src=$runsRoot,dst=/workspace/runs",
|
||||
'--mount', 'type=bind,src=D:\NDC_MISSIONCORE\runtime\derived\perception-p0-env-v1,dst=/opt/env,readonly',
|
||||
'--mount', 'type=bind,src=D:\NDC_MISSIONCORE\runtime\derived\perception-e15-media-pyav180-lz445-v1,dst=/opt/media,readonly',
|
||||
'--mount', 'type=bind,src=D:\NDC_MISSIONCORE\runtime\derived\perception-e3-opencv413092-v1,dst=/environment,readonly',
|
||||
'-e', 'PYTHONPATH=/opt/env:/opt/media:/environment/packages',
|
||||
'--entrypoint', '/bin/bash',
|
||||
$image, '-lc', $command
|
||||
)
|
||||
|
||||
& docker @dockerArguments
|
||||
$runnerExitCode = $LASTEXITCODE
|
||||
if ($runnerExitCode -ne 0 -and $runnerExitCode -ne 2) {
|
||||
throw "E46J runner crashed with exit code $runnerExitCode"
|
||||
}
|
||||
$result = Join-Path $runsRoot $runId
|
||||
$runtimePath = Join-Path $result 'runtime.json'
|
||||
if (-not $NoOverlay) {
|
||||
$intermediate = Join-Path $result 'raw-fisheye-yolox-overlay-intermediate.mp4'
|
||||
$overlay = Join-Path $result 'raw-fisheye-yolox-overlay.mp4'
|
||||
if (-not (Test-Path -LiteralPath $intermediate -PathType Leaf)) {
|
||||
throw 'E46J intermediate overlay is missing.'
|
||||
}
|
||||
& ffmpeg.exe -hide_banner -loglevel error -y -i $intermediate `
|
||||
-c:v libx264 -preset veryfast -crf 20 -r 4489000/448723 `
|
||||
-movflags +faststart -an $overlay
|
||||
if ($LASTEXITCODE -ne 0 -or
|
||||
-not (Test-Path -LiteralPath $overlay -PathType Leaf) -or
|
||||
(Get-Item -LiteralPath $overlay).Length -eq 0) {
|
||||
throw 'E46J final H.264 overlay transcode failed.'
|
||||
}
|
||||
$runtime = Get-Content -LiteralPath $runtimePath -Raw | ConvertFrom-Json
|
||||
$runtime.artifacts.PSObject.Properties.Remove('overlay_intermediate')
|
||||
$runtime.artifacts | Add-Member -NotePropertyName overlay -NotePropertyValue ([pscustomobject]@{
|
||||
file = 'raw-fisheye-yolox-overlay.mp4'
|
||||
byte_length = (Get-Item -LiteralPath $overlay).Length
|
||||
sha256 = Get-Sha256 $overlay
|
||||
codec = 'H.264'
|
||||
frame_rate = 10.003944527024467
|
||||
})
|
||||
$runtime | Add-Member -NotePropertyName visual_export -NotePropertyValue ([pscustomobject]@{
|
||||
provider = ((& ffmpeg.exe -version | Select-Object -First 1).Trim())
|
||||
source = 'MPEG-4 Part 2 intermediate produced by OpenCV VideoWriter'
|
||||
output = 'H.264 MP4 with faststart'
|
||||
excluded_from_core_latency = $true
|
||||
})
|
||||
$runtime | ConvertTo-Json -Depth 20 | Set-Content -LiteralPath $runtimePath -Encoding UTF8
|
||||
Remove-Item -LiteralPath $intermediate -Force
|
||||
}
|
||||
Write-Output "E46J_RUN_ID=$runId"
|
||||
Write-Output "E46J_RESULT_ROOT=$result"
|
||||
Get-Content -LiteralPath $runtimePath -Raw
|
||||
if ($runnerExitCode -eq 2) {
|
||||
exit 2
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$JobRoot,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$OutputPath,
|
||||
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 360
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-DDrivePath {
|
||||
param([string]$Path, [string]$Label)
|
||||
$fullPath = [IO.Path]::GetFullPath($Path)
|
||||
if ([IO.Path]::GetPathRoot($fullPath).TrimEnd("\") -ine "D:") {
|
||||
throw "$Label must be stored on D:"
|
||||
}
|
||||
return $fullPath
|
||||
}
|
||||
|
||||
function Assert-FreeSpace {
|
||||
param([string]$Phase, [int64]$RequiredAdditionalBytes = 0)
|
||||
$freeBytes = [int64](Get-PSDrive -Name D).Free
|
||||
$floorBytes = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Output (
|
||||
"DISK_GUARD PHASE={0} FREE_GIB={1} FLOOR_GIB={2}" -f
|
||||
$Phase,
|
||||
[math]::Round($freeBytes / 1GB, 3),
|
||||
$FreeGiBFloor
|
||||
)
|
||||
if ($freeBytes -lt ($floorBytes + $RequiredAdditionalBytes)) {
|
||||
throw "D: does not have the guarded replay reserve during $Phase"
|
||||
}
|
||||
}
|
||||
|
||||
$jobDirectory = Assert-DDrivePath (
|
||||
(Resolve-Path -LiteralPath $JobRoot).Path
|
||||
) "Job root"
|
||||
$output = Assert-DDrivePath $OutputPath "Output path"
|
||||
if (Test-Path -LiteralPath $output) {
|
||||
Write-Output "REPLAY_ALREADY_PRESENT=$output"
|
||||
exit 0
|
||||
}
|
||||
|
||||
$jobPath = Join-Path $jobDirectory "job.json"
|
||||
$job = Get-Content -LiteralPath $jobPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
$job.schema_version -ne "missioncore.compute-job/v1" -or
|
||||
$job.job_id -ne "recorded-camera-602ac89026ed12978619801d" -or
|
||||
$job.input.session_id -ne "20260720T065719Z_viewer_live" -or
|
||||
$job.input.source_id -ne "sensor.camera.right" -or
|
||||
[int]$job.input.segment_count -ne 4489
|
||||
) {
|
||||
throw "The requested job is not the immutable RAVNOVES00 right-camera source"
|
||||
}
|
||||
|
||||
$epochRoot = Join-Path $jobDirectory "input\camera\sensor.camera.right\epoch-1"
|
||||
$initPath = Join-Path $epochRoot "init.mp4"
|
||||
$segmentsRoot = Join-Path $epochRoot "segments"
|
||||
if (-not (Test-Path -LiteralPath $initPath -PathType Leaf)) {
|
||||
throw "Camera initialization segment is absent"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $segmentsRoot -PathType Container)) {
|
||||
throw "Camera segment directory is absent"
|
||||
}
|
||||
|
||||
$parent = Split-Path $output -Parent
|
||||
$null = New-Item -ItemType Directory -Path $parent -Force
|
||||
$temporary = Join-Path $parent (".{0}.{1}.partial" -f (Split-Path $output -Leaf), [Guid]::NewGuid().ToString("N"))
|
||||
$requiredBytes = [int64]$job.input.byte_length + 1GB
|
||||
Assert-FreeSpace "preflight" $requiredBytes
|
||||
|
||||
try {
|
||||
$destination = [IO.File]::Open(
|
||||
$temporary,
|
||||
[IO.FileMode]::CreateNew,
|
||||
[IO.FileAccess]::Write,
|
||||
[IO.FileShare]::None
|
||||
)
|
||||
try {
|
||||
$source = [IO.File]::OpenRead($initPath)
|
||||
try { $source.CopyTo($destination) } finally { $source.Dispose() }
|
||||
for ($sequence = 1; $sequence -le 4489; $sequence++) {
|
||||
$segment = Join-Path $segmentsRoot ("{0}.m4s" -f $sequence)
|
||||
if (-not (Test-Path -LiteralPath $segment -PathType Leaf)) {
|
||||
throw "Camera segment is absent: $sequence"
|
||||
}
|
||||
$source = [IO.File]::OpenRead($segment)
|
||||
try { $source.CopyTo($destination) } finally { $source.Dispose() }
|
||||
}
|
||||
$destination.Flush($true)
|
||||
}
|
||||
finally {
|
||||
$destination.Dispose()
|
||||
}
|
||||
|
||||
$probe = & ffprobe -v error -select_streams v:0 -count_frames `
|
||||
-show_entries stream=width,height,nb_read_frames `
|
||||
-of json $temporary | ConvertFrom-Json
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "ffprobe failed for the reconstructed camera source"
|
||||
}
|
||||
$stream = @($probe.streams)[0]
|
||||
if (
|
||||
[int]$stream.width -ne 800 -or
|
||||
[int]$stream.height -ne 600 -or
|
||||
[int]$stream.nb_read_frames -ne 4489
|
||||
) {
|
||||
throw "Reconstructed camera stream violates the immutable frame contract"
|
||||
}
|
||||
Move-Item -LiteralPath $temporary -Destination $output
|
||||
Assert-FreeSpace "published"
|
||||
Write-Output "REPLAY_PATH=$output"
|
||||
Write-Output "FRAME_COUNT=4489"
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $temporary) {
|
||||
Remove-Item -LiteralPath $temporary -Force
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
[application]
|
||||
enable-perf-measurement=1
|
||||
perf-measurement-interval-sec=5
|
||||
gie-kitti-output-dir=/workspace/output/detections
|
||||
kitti-track-output-dir=/workspace/output/tracks
|
||||
|
||||
[tiled-display]
|
||||
enable=0
|
||||
rows=1
|
||||
columns=1
|
||||
width=800
|
||||
height=600
|
||||
gpu-id=0
|
||||
|
||||
[source0]
|
||||
enable=1
|
||||
type=3
|
||||
num-sources=1
|
||||
uri=file:///workspace/input/right.mp4
|
||||
gpu-id=0
|
||||
|
||||
[streammux]
|
||||
gpu-id=0
|
||||
batch-size=1
|
||||
batched-push-timeout=40000
|
||||
width=800
|
||||
height=600
|
||||
live-source=0
|
||||
|
||||
[primary-gie]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
plugin-type=0
|
||||
batch-size=1
|
||||
gie-unique-id=1
|
||||
config-file=/workspace/package/runtime/e46e_trafficcamnet_rtdetr.txt
|
||||
bbox-border-color1=0.267;0.831;1.0;1.0
|
||||
bbox-border-color2=0.243;0.973;0.553;1.0
|
||||
bbox-border-color3=1.0;0.306;0.765;1.0
|
||||
bbox-border-color4=1.0;0.741;0.153;1.0
|
||||
|
||||
[tracker]
|
||||
enable=1
|
||||
tracker-width=960
|
||||
tracker-height=544
|
||||
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
|
||||
ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml
|
||||
gpu-id=0
|
||||
display-tracking-id=1
|
||||
compute-hw=1
|
||||
|
||||
[osd]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
border-width=3
|
||||
text-size=16
|
||||
text-color=1;1;1;1
|
||||
text-bg-color=0.08;0.08;0.08;0.9
|
||||
font=Arial
|
||||
display-bbox=1
|
||||
display-text=1
|
||||
|
||||
[sink0]
|
||||
enable=1
|
||||
type=3
|
||||
container=1
|
||||
codec=1
|
||||
enc-type=0
|
||||
sync=0
|
||||
qos=0
|
||||
bitrate=8000000
|
||||
profile=4
|
||||
output-file=/workspace/output/overlay.mp4
|
||||
source-id=0
|
||||
gpu-id=0
|
||||
|
||||
[tests]
|
||||
file-loop=0
|
||||
@@ -0,0 +1,5 @@
|
||||
background
|
||||
bicycle
|
||||
car
|
||||
person
|
||||
road_sign
|
||||
@@ -0,0 +1,26 @@
|
||||
[property]
|
||||
gpu-id=0
|
||||
onnx-file=/workspace/model/resnet50_trafficcamnet_rtdetr.fp16.onnx
|
||||
model-engine-file=/workspace/model/resnet50_trafficcamnet_rtdetr.fp16.onnx_b1_gpu0_fp16.engine
|
||||
labelfile-path=/workspace/package/runtime/e46e_trafficcamnet_labels.txt
|
||||
custom-lib-path=/workspace/package/runtime/libnvds_infercustomparser_tao.so
|
||||
parse-bbox-func-name=NvDsInferParseCustomDDETRTAO
|
||||
output-blob-names=pred_logits;pred_boxes
|
||||
infer-dims=3;544;960
|
||||
maintain-aspect-ratio=1
|
||||
net-scale-factor=0.00392156862745098
|
||||
offsets=0;0;0
|
||||
model-color-format=0
|
||||
network-mode=2
|
||||
network-type=0
|
||||
num-detected-classes=5
|
||||
cluster-mode=4
|
||||
output-tensor-meta=1
|
||||
workspace-size=1048576
|
||||
batch-size=1
|
||||
interval=0
|
||||
gie-unique-id=1
|
||||
|
||||
[class-attrs-all]
|
||||
pre-cluster-threshold=0.5
|
||||
topk=20
|
||||
@@ -0,0 +1,32 @@
|
||||
[property]
|
||||
gpu-id=0
|
||||
onnx-file=/workspace/model/resnet18_dashcamnet_pruned.onnx
|
||||
model-engine-file=/workspace/model/resnet18_dashcamnet_pruned.onnx_b1_gpu0_fp16.engine
|
||||
labelfile-path=/workspace/package/runtime/e46f_dashcamnet_labels.txt
|
||||
output-blob-names=output_bbox/BiasAdd:0;output_cov/Sigmoid:0
|
||||
infer-dims=3;544;960
|
||||
maintain-aspect-ratio=0
|
||||
net-scale-factor=0.00392156862745098
|
||||
offsets=0;0;0
|
||||
model-color-format=0
|
||||
network-mode=2
|
||||
network-type=0
|
||||
num-detected-classes=4
|
||||
cluster-mode=2
|
||||
output-tensor-meta=0
|
||||
workspace-size=1048576
|
||||
batch-size=1
|
||||
interval=0
|
||||
gie-unique-id=1
|
||||
|
||||
[class-attrs-all]
|
||||
topk=20
|
||||
nms-iou-threshold=0.5
|
||||
pre-cluster-threshold=0.2
|
||||
roi-top-offset=0
|
||||
roi-bottom-offset=0
|
||||
|
||||
[class-attrs-0]
|
||||
topk=20
|
||||
nms-iou-threshold=0.5
|
||||
pre-cluster-threshold=0.4
|
||||
@@ -0,0 +1,4 @@
|
||||
car
|
||||
bicycle
|
||||
person
|
||||
road_sign
|
||||
@@ -0,0 +1,78 @@
|
||||
[application]
|
||||
enable-perf-measurement=1
|
||||
perf-measurement-interval-sec=5
|
||||
gie-kitti-output-dir=/workspace/output/detections
|
||||
kitti-track-output-dir=/workspace/output/tracks
|
||||
|
||||
[tiled-display]
|
||||
enable=0
|
||||
rows=1
|
||||
columns=1
|
||||
width=800
|
||||
height=600
|
||||
gpu-id=0
|
||||
|
||||
[source0]
|
||||
enable=1
|
||||
type=3
|
||||
num-sources=1
|
||||
uri=file:///workspace/input/right.mp4
|
||||
gpu-id=0
|
||||
|
||||
[streammux]
|
||||
gpu-id=0
|
||||
batch-size=1
|
||||
batched-push-timeout=40000
|
||||
width=800
|
||||
height=600
|
||||
live-source=0
|
||||
|
||||
[primary-gie]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
plugin-type=0
|
||||
batch-size=1
|
||||
gie-unique-id=1
|
||||
config-file=/workspace/package/runtime/e46f_dashcamnet_detectnet.txt
|
||||
bbox-border-color0=0.243;0.973;0.553;1.0
|
||||
bbox-border-color1=0.267;0.831;1.0;1.0
|
||||
bbox-border-color2=1.0;0.306;0.765;1.0
|
||||
bbox-border-color3=1.0;0.741;0.153;1.0
|
||||
|
||||
[tracker]
|
||||
enable=1
|
||||
tracker-width=960
|
||||
tracker-height=544
|
||||
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
|
||||
ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml
|
||||
gpu-id=0
|
||||
display-tracking-id=1
|
||||
compute-hw=1
|
||||
|
||||
[osd]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
border-width=3
|
||||
text-size=16
|
||||
text-color=1;1;1;1
|
||||
text-bg-color=0.08;0.08;0.08;0.9
|
||||
font=Arial
|
||||
display-bbox=1
|
||||
display-text=1
|
||||
|
||||
[sink0]
|
||||
enable=1
|
||||
type=3
|
||||
container=1
|
||||
codec=1
|
||||
enc-type=0
|
||||
sync=0
|
||||
qos=0
|
||||
bitrate=8000000
|
||||
profile=4
|
||||
output-file=/workspace/output/overlay.mp4
|
||||
source-id=0
|
||||
gpu-id=0
|
||||
|
||||
[tests]
|
||||
file-loop=0
|
||||
@@ -0,0 +1,78 @@
|
||||
[application]
|
||||
enable-perf-measurement=1
|
||||
perf-measurement-interval-sec=5
|
||||
gie-kitti-output-dir=/workspace/output/detections
|
||||
kitti-track-output-dir=/workspace/output/tracks
|
||||
|
||||
[tiled-display]
|
||||
enable=0
|
||||
rows=1
|
||||
columns=1
|
||||
width=960
|
||||
height=544
|
||||
gpu-id=0
|
||||
|
||||
[source0]
|
||||
enable=1
|
||||
type=3
|
||||
num-sources=1
|
||||
uri=file:///workspace/input/view.mp4
|
||||
gpu-id=0
|
||||
|
||||
[streammux]
|
||||
gpu-id=0
|
||||
batch-size=1
|
||||
batched-push-timeout=40000
|
||||
width=960
|
||||
height=544
|
||||
live-source=0
|
||||
|
||||
[primary-gie]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
plugin-type=0
|
||||
batch-size=1
|
||||
gie-unique-id=1
|
||||
config-file=/workspace/package/runtime/e46f_dashcamnet_detectnet.txt
|
||||
bbox-border-color0=0.243;0.973;0.553;1.0
|
||||
bbox-border-color1=0.267;0.831;1.0;1.0
|
||||
bbox-border-color2=1.0;0.306;0.765;1.0
|
||||
bbox-border-color3=1.0;0.741;0.153;1.0
|
||||
|
||||
[tracker]
|
||||
enable=1
|
||||
tracker-width=960
|
||||
tracker-height=544
|
||||
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
|
||||
ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml
|
||||
gpu-id=0
|
||||
display-tracking-id=1
|
||||
compute-hw=1
|
||||
|
||||
[osd]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
border-width=3
|
||||
text-size=16
|
||||
text-color=1;1;1;1
|
||||
text-bg-color=0.08;0.08;0.08;0.9
|
||||
font=Arial
|
||||
display-bbox=1
|
||||
display-text=1
|
||||
|
||||
[sink0]
|
||||
enable=1
|
||||
type=3
|
||||
container=1
|
||||
codec=1
|
||||
enc-type=0
|
||||
sync=0
|
||||
qos=0
|
||||
bitrate=6000000
|
||||
profile=4
|
||||
output-file=/workspace/output/overlay.mp4
|
||||
source-id=0
|
||||
gpu-id=0
|
||||
|
||||
[tests]
|
||||
file-loop=0
|
||||
@@ -0,0 +1,22 @@
|
||||
[property]
|
||||
output-width=960
|
||||
output-height=544
|
||||
num-batch-buffers=1
|
||||
cuda-memory-type=2
|
||||
|
||||
[surface0]
|
||||
projection-type=4
|
||||
surface-index=0
|
||||
width=960
|
||||
height=544
|
||||
yaw=0.0
|
||||
pitch=0.0
|
||||
roll=0.0
|
||||
rot-axes=YXZ
|
||||
focal-length=194.59817287616025;194.57531427932872
|
||||
distortion=-0.023164451386679667;-0.0014974198594105452;-0.001039213149441563;-0.000035237331915978814
|
||||
src-x0=396.31861150187996
|
||||
src-y0=301.49644357408005
|
||||
dst-focal-length=402.76782296509447;402.76782296509447
|
||||
dst-principal-point=479.5;271.5
|
||||
cuda-address-mode=1
|
||||
@@ -0,0 +1,22 @@
|
||||
[property]
|
||||
output-width=960
|
||||
output-height=544
|
||||
num-batch-buffers=1
|
||||
cuda-memory-type=2
|
||||
|
||||
[surface0]
|
||||
projection-type=4
|
||||
surface-index=0
|
||||
width=960
|
||||
height=544
|
||||
yaw=270.0
|
||||
pitch=0.0
|
||||
roll=0.0
|
||||
rot-axes=YXZ
|
||||
focal-length=194.59817287616025;194.57531427932872
|
||||
distortion=-0.023164451386679667;-0.0014974198594105452;-0.001039213149441563;-0.000035237331915978814
|
||||
src-x0=396.31861150187996
|
||||
src-y0=301.49644357408005
|
||||
dst-focal-length=402.76782296509447;402.76782296509447
|
||||
dst-principal-point=479.5;271.5
|
||||
cuda-address-mode=1
|
||||
@@ -0,0 +1,22 @@
|
||||
[property]
|
||||
output-width=960
|
||||
output-height=544
|
||||
num-batch-buffers=1
|
||||
cuda-memory-type=2
|
||||
|
||||
[surface0]
|
||||
projection-type=4
|
||||
surface-index=0
|
||||
width=960
|
||||
height=544
|
||||
yaw=90.0
|
||||
pitch=0.0
|
||||
roll=0.0
|
||||
rot-axes=YXZ
|
||||
focal-length=194.59817287616025;194.57531427932872
|
||||
distortion=-0.023164451386679667;-0.0014974198594105452;-0.001039213149441563;-0.000035237331915978814
|
||||
src-x0=396.31861150187996
|
||||
src-y0=301.49644357408005
|
||||
dst-focal-length=402.76782296509447;402.76782296509447
|
||||
dst-principal-point=479.5;271.5
|
||||
cuda-address-mode=1
|
||||
@@ -0,0 +1,78 @@
|
||||
[application]
|
||||
enable-perf-measurement=1
|
||||
perf-measurement-interval-sec=5
|
||||
gie-kitti-output-dir=/workspace/output/detections
|
||||
kitti-track-output-dir=/workspace/output/tracks
|
||||
|
||||
[tiled-display]
|
||||
enable=0
|
||||
rows=1
|
||||
columns=1
|
||||
width=960
|
||||
height=544
|
||||
gpu-id=0
|
||||
|
||||
[source0]
|
||||
enable=1
|
||||
type=3
|
||||
num-sources=1
|
||||
uri=file:///workspace/input/view.mp4
|
||||
gpu-id=0
|
||||
|
||||
[streammux]
|
||||
gpu-id=0
|
||||
batch-size=1
|
||||
batched-push-timeout=40000
|
||||
width=960
|
||||
height=544
|
||||
live-source=0
|
||||
|
||||
[primary-gie]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
plugin-type=0
|
||||
batch-size=1
|
||||
gie-unique-id=1
|
||||
config-file=/workspace/package/runtime/e46e_trafficcamnet_rtdetr.txt
|
||||
bbox-border-color1=0.267;0.831;1.0;1.0
|
||||
bbox-border-color2=0.243;0.973;0.553;1.0
|
||||
bbox-border-color3=1.0;0.306;0.765;1.0
|
||||
bbox-border-color4=1.0;0.741;0.153;1.0
|
||||
|
||||
[tracker]
|
||||
enable=1
|
||||
tracker-width=960
|
||||
tracker-height=544
|
||||
ll-lib-file=/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so
|
||||
ll-config-file=/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_perf.yml
|
||||
gpu-id=0
|
||||
display-tracking-id=1
|
||||
compute-hw=1
|
||||
|
||||
[osd]
|
||||
enable=1
|
||||
gpu-id=0
|
||||
border-width=3
|
||||
text-size=16
|
||||
text-color=1;1;1;1
|
||||
text-bg-color=0.08;0.08;0.08;0.9
|
||||
font=Arial
|
||||
display-bbox=1
|
||||
display-text=1
|
||||
|
||||
[sink0]
|
||||
enable=1
|
||||
type=3
|
||||
container=1
|
||||
codec=1
|
||||
enc-type=0
|
||||
sync=0
|
||||
qos=0
|
||||
bitrate=6000000
|
||||
profile=4
|
||||
output-file=/workspace/output/overlay.mp4
|
||||
source-id=0
|
||||
gpu-id=0
|
||||
|
||||
[tests]
|
||||
file-loop=0
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Consolidate raw DeepStream/NvDCF output into immutable E46E evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46e_ready_stack import build_e46e_ready_stack
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-job", type=Path, required=True)
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46e_ready_stack(
|
||||
source_job_root=args.source_job,
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"metrics": result["report"]["metrics"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Consolidate raw DashCamNet/NvDCF output into immutable E46F evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46f_dashcam_bakeoff import build_e46f_dashcam_bakeoff
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-job", type=Path, required=True)
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46f_dashcam_bakeoff(
|
||||
source_job_root=args.source_job,
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result["result_id"],
|
||||
"result_root": str(result["result_root"]),
|
||||
"metrics": result["report"]["metrics"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze raw NVIDIA E46G output as immutable Mission Core evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46g_rectified_detector_bakeoff import (
|
||||
build_e46g_rectified_detector_bakeoff,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-job", type=Path, required=True)
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46g_rectified_detector_bakeoff(
|
||||
source_job_root=args.source_job,
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(result["result_root"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze raw NVIDIA E46H output as immutable Mission Core evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46h_full_rectified_front_replay import (
|
||||
build_e46h_full_rectified_front_replay,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-job", type=Path, required=True)
|
||||
parser.add_argument("--raw-root", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46h_full_rectified_front_replay(
|
||||
source_job_root=args.source_job,
|
||||
raw_root=args.raw_root,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(result["result_root"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,706 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the frozen one-pass YOLOX-S realtime gate on the full K1 RIGHT fisheye.
|
||||
|
||||
The runner deliberately performs exactly one detector request per decoded source
|
||||
frame. It does not rectify, crop, tile, track, hold, stitch or use route-specific
|
||||
filters. H.264 overlay encoding is measured separately from the detector path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections import Counter
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
PROFILE_SCHEMA = "missioncore.e46j-raw-fisheye-realtime-profile/v1"
|
||||
RUNTIME_SCHEMA = "missioncore.e46j-raw-fisheye-realtime-runtime/v1"
|
||||
FRAME_SCHEMA = "missioncore.e46j-raw-fisheye-realtime-frame/v1"
|
||||
|
||||
COCO_CLASSES = (
|
||||
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
|
||||
"truck", "boat", "traffic light", "fire hydrant", "stop sign",
|
||||
"parking meter", "bench", "bird", "cat", "dog", "horse", "sheep",
|
||||
"cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella",
|
||||
"handbag", "tie", "suitcase", "frisbee", "skis", "snowboard",
|
||||
"sports ball", "kite", "baseball bat", "baseball glove", "skateboard",
|
||||
"surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork",
|
||||
"knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
|
||||
"broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair",
|
||||
"couch", "potted plant", "bed", "dining table", "toilet", "tv",
|
||||
"laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
|
||||
"oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
|
||||
"scissors", "teddy bear", "hair drier", "toothbrush",
|
||||
)
|
||||
|
||||
|
||||
def arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--mask", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--triton-url", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--max-frames", type=int, default=0)
|
||||
parser.add_argument("--overlay", action="store_true")
|
||||
parser.add_argument("--telemetry-interval", type=float, default=0.5)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.resolve(strict=True).read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON object expected: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value, sort_keys=True, separators=(",", ":"), allow_nan=False
|
||||
).encode()
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def distribution(values: list[float]) -> dict[str, float]:
|
||||
if not values:
|
||||
return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "maximum": 0.0}
|
||||
ordered = sorted(values)
|
||||
|
||||
def percentile(fraction: float) -> float:
|
||||
index = (len(ordered) - 1) * fraction
|
||||
lower = math.floor(index)
|
||||
upper = math.ceil(index)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
ratio = index - lower
|
||||
return ordered[lower] * (1.0 - ratio) + ordered[upper] * ratio
|
||||
|
||||
return {
|
||||
"mean": round(statistics.fmean(ordered), 6),
|
||||
"p50": round(percentile(0.5), 6),
|
||||
"p95": round(percentile(0.95), 6),
|
||||
"maximum": round(max(ordered), 6),
|
||||
}
|
||||
|
||||
|
||||
def validate_profile(profile: dict[str, Any], source_sha256: str) -> None:
|
||||
source = profile.get("source")
|
||||
detector = profile.get("detector")
|
||||
detection = profile.get("detection")
|
||||
acceptance = profile.get("acceptance")
|
||||
if (
|
||||
profile.get("schema_version") != PROFILE_SCHEMA
|
||||
or not isinstance(source, dict)
|
||||
or source.get("camera_source_id") != "sensor.camera.right"
|
||||
or source.get("stream_sha256") != source_sha256
|
||||
or source.get("resolution") != [800, 600]
|
||||
or source.get("calibration_model") != "KB4"
|
||||
or not isinstance(detector, dict)
|
||||
or detector.get("id") != "yolox_s"
|
||||
or detector.get("input_shape") != [1, 3, 640, 640]
|
||||
or detector.get("single_inference_per_source_frame") is not True
|
||||
or not isinstance(detection, dict)
|
||||
or detection.get("custom_detector_logic") is not False
|
||||
or detection.get("route_specific_filtering") is not False
|
||||
or not isinstance(acceptance, dict)
|
||||
or acceptance.get("require_full_raw_fov") is not True
|
||||
):
|
||||
raise RuntimeError("E46J profile contract changed")
|
||||
|
||||
|
||||
def load_mask(path: Path) -> np.ndarray:
|
||||
from PIL import Image
|
||||
|
||||
mask = np.asarray(Image.open(path.resolve(strict=True)).convert("L")) > 0
|
||||
if mask.shape != (600, 800) or not np.any(mask):
|
||||
raise RuntimeError("E46J valid-FOV mask changed")
|
||||
return mask
|
||||
|
||||
|
||||
def preprocess(
|
||||
image_bgr: np.ndarray, mask: np.ndarray, profile: dict[str, Any]
|
||||
) -> np.ndarray:
|
||||
import cv2
|
||||
|
||||
target_height = int(profile["detector"]["input_shape"][2])
|
||||
target_width = int(profile["detector"]["input_shape"][3])
|
||||
height, width = image_bgr.shape[:2]
|
||||
ratio = min(target_height / height, target_width / width)
|
||||
resized_width = int(width * ratio)
|
||||
resized_height = int(height * ratio)
|
||||
fill = int(profile["preprocessing"]["valid_fov_fill_value"])
|
||||
masked = np.where(mask[..., None], image_bgr, fill).astype(np.uint8)
|
||||
resized = cv2.resize(
|
||||
masked, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR
|
||||
)
|
||||
canvas = np.full((target_height, target_width, 3), fill, dtype=np.uint8)
|
||||
canvas[:resized_height, :resized_width] = resized
|
||||
return np.ascontiguousarray(canvas.transpose(2, 0, 1), dtype=np.float32)[None]
|
||||
|
||||
|
||||
class TritonHttpClient:
|
||||
"""Minimal persistent Triton HTTP client for one sequential camera stream."""
|
||||
|
||||
def __init__(self, url: str, model: dict[str, Any]) -> None:
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
if parsed.scheme != "http" or not parsed.hostname:
|
||||
raise RuntimeError("E46J requires an explicit HTTP Triton endpoint")
|
||||
self.model = model
|
||||
self.path = f"{parsed.path.rstrip('/')}/v2/models/{model['id']}/infer"
|
||||
self.connection = http.client.HTTPConnection(
|
||||
parsed.hostname,
|
||||
parsed.port or 80,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
self.connection.close()
|
||||
|
||||
def infer(self, tensor: np.ndarray) -> np.ndarray:
|
||||
contiguous = np.ascontiguousarray(tensor, dtype=np.float32)
|
||||
binary = contiguous.tobytes()
|
||||
header = {
|
||||
"inputs": [
|
||||
{
|
||||
"name": self.model["input_name"],
|
||||
"shape": list(contiguous.shape),
|
||||
"datatype": "FP32",
|
||||
"parameters": {"binary_data_size": len(binary)},
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": self.model["output_name"],
|
||||
"parameters": {"binary_data": True},
|
||||
}
|
||||
],
|
||||
}
|
||||
encoded = canonical_json(header)
|
||||
self.connection.request(
|
||||
"POST",
|
||||
self.path,
|
||||
body=encoded + binary,
|
||||
headers={
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Inference-Header-Content-Length": str(len(encoded)),
|
||||
},
|
||||
)
|
||||
response = self.connection.getresponse()
|
||||
payload = response.read()
|
||||
if response.status != 200:
|
||||
raise RuntimeError(
|
||||
f"E46J Triton inference failed: HTTP {response.status}: "
|
||||
f"{payload[:512]!r}"
|
||||
)
|
||||
header_length_value = response.getheader("Inference-Header-Content-Length")
|
||||
if not header_length_value:
|
||||
raise RuntimeError("E46J Triton output header length is missing")
|
||||
header_length = int(header_length_value)
|
||||
descriptor = json.loads(payload[:header_length])["outputs"][0]
|
||||
if (
|
||||
descriptor["name"] != self.model["output_name"]
|
||||
or descriptor["datatype"] != "FP32"
|
||||
):
|
||||
raise RuntimeError("E46J Triton output descriptor changed")
|
||||
shape = tuple(int(value) for value in descriptor["shape"])
|
||||
array = np.frombuffer(payload[header_length:], dtype="<f4")
|
||||
if array.size != math.prod(shape):
|
||||
raise RuntimeError("E46J Triton output byte length changed")
|
||||
return array.reshape(shape)
|
||||
|
||||
|
||||
def decode_yolox(output: np.ndarray) -> np.ndarray:
|
||||
predictions = output.copy()
|
||||
grids: list[np.ndarray] = []
|
||||
strides: list[np.ndarray] = []
|
||||
for stride in (8, 16, 32):
|
||||
height = 640 // stride
|
||||
width = 640 // stride
|
||||
yv, xv = np.meshgrid(np.arange(height), np.arange(width), indexing="ij")
|
||||
grids.append(np.stack((xv, yv), axis=2).reshape(1, -1, 2))
|
||||
strides.append(np.full((1, height * width, 1), stride))
|
||||
grid = np.concatenate(grids, axis=1)
|
||||
expanded_strides = np.concatenate(strides, axis=1)
|
||||
predictions[..., :2] = (predictions[..., :2] + grid) * expanded_strides
|
||||
predictions[..., 2:4] = np.exp(predictions[..., 2:4]) * expanded_strides
|
||||
return predictions
|
||||
|
||||
|
||||
def box_iou(one: np.ndarray, many: np.ndarray) -> np.ndarray:
|
||||
if many.size == 0:
|
||||
return np.zeros((0,), dtype=np.float32)
|
||||
top_left = np.maximum(one[:2], many[:, :2])
|
||||
bottom_right = np.minimum(one[2:], many[:, 2:])
|
||||
intersection = np.prod(np.maximum(0.0, bottom_right - top_left), axis=1)
|
||||
one_area = max(0.0, float(one[2] - one[0])) * max(
|
||||
0.0, float(one[3] - one[1])
|
||||
)
|
||||
many_area = np.maximum(0.0, many[:, 2] - many[:, 0]) * np.maximum(
|
||||
0.0, many[:, 3] - many[:, 1]
|
||||
)
|
||||
union = one_area + many_area - intersection
|
||||
return np.divide(
|
||||
intersection, union, out=np.zeros_like(intersection), where=union > 0
|
||||
)
|
||||
|
||||
|
||||
def nms(boxes: np.ndarray, scores: np.ndarray, threshold: float) -> list[int]:
|
||||
order = scores.argsort()[::-1]
|
||||
keep: list[int] = []
|
||||
while order.size:
|
||||
index = int(order[0])
|
||||
keep.append(index)
|
||||
overlaps = box_iou(boxes[index], boxes[order[1:]])
|
||||
order = order[np.where(overlaps <= threshold)[0] + 1]
|
||||
return keep
|
||||
|
||||
|
||||
def valid_fraction(
|
||||
box: np.ndarray, integral: np.ndarray
|
||||
) -> tuple[float, bool, float]:
|
||||
height = integral.shape[0] - 1
|
||||
width = integral.shape[1] - 1
|
||||
x1 = int(np.clip(math.floor(float(box[0])), 0, width))
|
||||
y1 = int(np.clip(math.floor(float(box[1])), 0, height))
|
||||
x2 = int(np.clip(math.ceil(float(box[2])), 0, width))
|
||||
y2 = int(np.clip(math.ceil(float(box[3])), 0, height))
|
||||
area = float(max(0, x2 - x1) * max(0, y2 - y1))
|
||||
if area <= 0:
|
||||
return 0.0, False, 0.0
|
||||
inside = integral[y2, x2] - integral[y1, x2] - integral[y2, x1] + integral[
|
||||
y1, x1
|
||||
]
|
||||
center_x = int(
|
||||
np.clip(round((float(box[0]) + float(box[2])) / 2.0), 0, width - 1)
|
||||
)
|
||||
center_y = int(
|
||||
np.clip(round((float(box[1]) + float(box[3])) / 2.0), 0, height - 1)
|
||||
)
|
||||
center_inside = bool(
|
||||
integral[center_y + 1, center_x + 1]
|
||||
- integral[center_y, center_x + 1]
|
||||
- integral[center_y + 1, center_x]
|
||||
+ integral[center_y, center_x]
|
||||
)
|
||||
return float(inside) / area, center_inside, area
|
||||
|
||||
|
||||
def detections(
|
||||
output: np.ndarray, profile: dict[str, Any], mask: np.ndarray
|
||||
) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
prediction = decode_yolox(output)[0]
|
||||
boxes = prediction[:, :4]
|
||||
boxes_xyxy = np.empty_like(boxes)
|
||||
boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2.0
|
||||
boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2.0
|
||||
boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2.0
|
||||
boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2.0
|
||||
source_height, source_width = mask.shape
|
||||
ratio = min(640 / source_height, 640 / source_width)
|
||||
boxes_xyxy /= ratio
|
||||
class_scores = prediction[:, 4:5] * prediction[:, 5:]
|
||||
class_ids = class_scores.argmax(axis=1)
|
||||
scores = class_scores[np.arange(class_scores.shape[0]), class_ids]
|
||||
detection = profile["detection"]
|
||||
target_ids = set(int(value) for value in detection["target_class_ids"])
|
||||
candidate_mask = np.logical_and(
|
||||
scores >= float(detection["minimum_score"]),
|
||||
np.isin(class_ids, list(target_ids)),
|
||||
)
|
||||
candidate_boxes = boxes_xyxy[candidate_mask]
|
||||
candidate_scores = scores[candidate_mask]
|
||||
candidate_classes = class_ids[candidate_mask]
|
||||
integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1)
|
||||
result: list[dict[str, Any]] = []
|
||||
rejected: Counter[str] = Counter()
|
||||
for class_id in sorted(target_ids):
|
||||
indices = np.where(candidate_classes == class_id)[0]
|
||||
if not indices.size:
|
||||
continue
|
||||
keep = nms(
|
||||
candidate_boxes[indices],
|
||||
candidate_scores[indices],
|
||||
float(detection["nms_iou_threshold"]),
|
||||
)
|
||||
for selected in indices[keep]:
|
||||
box = candidate_boxes[selected].copy()
|
||||
box[[0, 2]] = np.clip(box[[0, 2]], 0, source_width)
|
||||
box[[1, 3]] = np.clip(box[[1, 3]], 0, source_height)
|
||||
fraction, center_inside, area = valid_fraction(box, integral)
|
||||
if area < float(detection["minimum_box_area_pixels"]):
|
||||
rejected["small_box"] += 1
|
||||
continue
|
||||
if area / float(source_width * source_height) > float(
|
||||
detection["maximum_box_area_fraction"]
|
||||
):
|
||||
rejected["large_box"] += 1
|
||||
continue
|
||||
if fraction < float(detection["minimum_valid_fov_fraction"]):
|
||||
rejected["outside_valid_fov"] += 1
|
||||
continue
|
||||
if detection["require_center_inside_valid_fov"] and not center_inside:
|
||||
rejected["center_outside_valid_fov"] += 1
|
||||
continue
|
||||
result.append(
|
||||
{
|
||||
"class_id": int(class_id),
|
||||
"label": COCO_CLASSES[int(class_id)],
|
||||
"score": round(float(candidate_scores[selected]), 9),
|
||||
"bbox_xyxy": [round(float(value), 6) for value in box],
|
||||
"valid_fov_fraction": round(fraction, 6),
|
||||
}
|
||||
)
|
||||
result.sort(key=lambda item: (-float(item["score"]), int(item["class_id"])))
|
||||
return result, dict(rejected)
|
||||
|
||||
|
||||
def draw_overlay(image: np.ndarray, rows: list[dict[str, Any]]) -> np.ndarray:
|
||||
import cv2
|
||||
|
||||
for row in rows:
|
||||
x1, y1, x2, y2 = (int(round(value)) for value in row["bbox_xyxy"])
|
||||
label = f"{row['label']} {float(row['score']):.2f}"
|
||||
cv2.rectangle(image, (x1, y1), (x2, y2), (248, 248, 248), 2)
|
||||
(text_width, text_height), baseline = cv2.getTextSize(
|
||||
label, cv2.FONT_HERSHEY_SIMPLEX, 0.45, 1
|
||||
)
|
||||
top = max(0, y1 - text_height - baseline - 6)
|
||||
cv2.rectangle(
|
||||
image,
|
||||
(x1, top),
|
||||
(min(image.shape[1] - 1, x1 + text_width + 8), y1),
|
||||
(18, 18, 18),
|
||||
-1,
|
||||
)
|
||||
cv2.putText(
|
||||
image,
|
||||
label,
|
||||
(x1 + 4, max(text_height + 1, y1 - baseline - 3)),
|
||||
cv2.FONT_HERSHEY_SIMPLEX,
|
||||
0.45,
|
||||
(248, 248, 248),
|
||||
1,
|
||||
cv2.LINE_AA,
|
||||
)
|
||||
return image
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, interval: float) -> None:
|
||||
self.interval = interval
|
||||
self.samples: list[dict[str, float]] = []
|
||||
self.stop_event = threading.Event()
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def __enter__(self) -> Telemetry:
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: object) -> None:
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=5)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self.stop_event.is_set():
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"nvidia-smi",
|
||||
"--query-gpu=utilization.gpu,memory.used,power.draw,temperature.gpu",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
values = [float(value.strip()) for value in completed.stdout.split(",")]
|
||||
self.samples.append(
|
||||
{
|
||||
"gpu_utilization_percent": values[0],
|
||||
"gpu_memory_used_mib": values[1],
|
||||
"gpu_power_watts": values[2],
|
||||
"gpu_temperature_celsius": values[3],
|
||||
}
|
||||
)
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
self.stop_event.wait(self.interval)
|
||||
|
||||
|
||||
def telemetry_summary(samples: list[dict[str, float]]) -> dict[str, object]:
|
||||
result: dict[str, object] = {"sample_count": len(samples)}
|
||||
for key in (
|
||||
"gpu_utilization_percent",
|
||||
"gpu_memory_used_mib",
|
||||
"gpu_power_watts",
|
||||
"gpu_temperature_celsius",
|
||||
):
|
||||
result[key] = distribution([row[key] for row in samples])
|
||||
return result
|
||||
|
||||
|
||||
def run(args: argparse.Namespace) -> dict[str, Any]:
|
||||
import av
|
||||
|
||||
source = args.input.resolve(strict=True)
|
||||
profile_path = args.profile.resolve(strict=True)
|
||||
mask_path = args.mask.resolve(strict=True)
|
||||
output = args.output.expanduser().absolute()
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
profile = read_object(profile_path)
|
||||
source_sha = sha256(source)
|
||||
validate_profile(profile, source_sha)
|
||||
mask = load_mask(mask_path)
|
||||
mask_sha = sha256(mask_path)
|
||||
|
||||
container = av.open(str(source))
|
||||
video_stream = container.streams.video[0]
|
||||
frame_rate = float(video_stream.average_rate)
|
||||
if not math.isclose(frame_rate, float(profile["source"]["frame_rate"]), abs_tol=1e-6):
|
||||
raise RuntimeError(
|
||||
"E46J source frame rate changed: "
|
||||
f"profile={profile['source']['frame_rate']}, actual={frame_rate}, "
|
||||
f"average_rate={video_stream.average_rate}, time_base={video_stream.time_base}"
|
||||
)
|
||||
|
||||
overlay_path = output / "raw-fisheye-yolox-overlay-intermediate.mp4"
|
||||
overlay_writer = None
|
||||
if args.overlay:
|
||||
import cv2
|
||||
|
||||
overlay_writer = cv2.VideoWriter(
|
||||
str(overlay_path),
|
||||
cv2.VideoWriter_fourcc(*"mp4v"),
|
||||
frame_rate,
|
||||
(800, 600),
|
||||
)
|
||||
if not overlay_writer.isOpened():
|
||||
raise RuntimeError("E46J intermediate overlay encoder is unavailable")
|
||||
|
||||
frame_path = output / "frames.jsonl"
|
||||
frame_stream = frame_path.open("x", encoding="utf-8")
|
||||
class_counts: Counter[str] = Counter()
|
||||
rejected_counts: Counter[str] = Counter()
|
||||
latencies: dict[str, list[float]] = {
|
||||
"preprocess_ms": [],
|
||||
"inference_request_ms": [],
|
||||
"postprocess_ms": [],
|
||||
"core_path_ms": [],
|
||||
"overlay_encode_ms": [],
|
||||
}
|
||||
zero_detection_frames = 0
|
||||
maximum_detections = 0
|
||||
failed_frames = 0
|
||||
processed_frames = 0
|
||||
run_started_utc = datetime.now(UTC)
|
||||
wall_started = time.perf_counter()
|
||||
warm_tensor = np.full((1, 3, 640, 640), 114.0, dtype=np.float32)
|
||||
triton = TritonHttpClient(args.triton_url, profile["detector"])
|
||||
triton.infer(warm_tensor)
|
||||
|
||||
with Telemetry(args.telemetry_interval) as telemetry:
|
||||
try:
|
||||
for decoded in container.decode(video_stream):
|
||||
if args.max_frames and processed_frames >= args.max_frames:
|
||||
break
|
||||
frame_index = processed_frames
|
||||
image = decoded.to_ndarray(format="bgr24")
|
||||
if image.shape != (600, 800, 3):
|
||||
raise RuntimeError(f"E46J source raster changed at frame {frame_index}")
|
||||
core_started = time.perf_counter()
|
||||
preprocess_started = core_started
|
||||
tensor = preprocess(image, mask, profile)
|
||||
preprocess_ms = (time.perf_counter() - preprocess_started) * 1000.0
|
||||
inference_started = time.perf_counter()
|
||||
raw_output = triton.infer(tensor)
|
||||
inference_ms = (time.perf_counter() - inference_started) * 1000.0
|
||||
postprocess_started = time.perf_counter()
|
||||
rows, rejected = detections(raw_output, profile, mask)
|
||||
postprocess_ms = (time.perf_counter() - postprocess_started) * 1000.0
|
||||
core_path_ms = (time.perf_counter() - core_started) * 1000.0
|
||||
latencies["preprocess_ms"].append(preprocess_ms)
|
||||
latencies["inference_request_ms"].append(inference_ms)
|
||||
latencies["postprocess_ms"].append(postprocess_ms)
|
||||
latencies["core_path_ms"].append(core_path_ms)
|
||||
class_counts.update(str(row["label"]) for row in rows)
|
||||
rejected_counts.update(rejected)
|
||||
maximum_detections = max(maximum_detections, len(rows))
|
||||
if not rows:
|
||||
zero_detection_frames += 1
|
||||
|
||||
overlay_ms = 0.0
|
||||
if overlay_writer is not None:
|
||||
overlay_started = time.perf_counter()
|
||||
overlay = draw_overlay(image.copy(), rows)
|
||||
overlay_writer.write(overlay)
|
||||
overlay_ms = (time.perf_counter() - overlay_started) * 1000.0
|
||||
latencies["overlay_encode_ms"].append(overlay_ms)
|
||||
frame_stream.write(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": FRAME_SCHEMA,
|
||||
"frame_index": frame_index,
|
||||
"session_seconds": round(frame_index / frame_rate, 6),
|
||||
"detections": rows,
|
||||
"rejected": rejected,
|
||||
"latency_ms": {
|
||||
"preprocess": round(preprocess_ms, 6),
|
||||
"inference_request": round(inference_ms, 6),
|
||||
"postprocess": round(postprocess_ms, 6),
|
||||
"core_path": round(core_path_ms, 6),
|
||||
"overlay_encode": round(overlay_ms, 6),
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
processed_frames += 1
|
||||
except BaseException:
|
||||
failed_frames += 1
|
||||
raise
|
||||
finally:
|
||||
frame_stream.flush()
|
||||
os.fsync(frame_stream.fileno())
|
||||
frame_stream.close()
|
||||
container.close()
|
||||
triton.close()
|
||||
if overlay_writer is not None:
|
||||
overlay_writer.release()
|
||||
|
||||
wall_seconds = time.perf_counter() - wall_started
|
||||
run_completed_utc = datetime.now(UTC)
|
||||
core = distribution(latencies["core_path_ms"])
|
||||
inference = distribution(latencies["inference_request_ms"])
|
||||
acceptance = profile["acceptance"]
|
||||
expected_frames = (
|
||||
min(int(args.max_frames), int(profile["source"]["frame_count"]))
|
||||
if args.max_frames
|
||||
else int(profile["source"]["frame_count"])
|
||||
)
|
||||
checks = {
|
||||
"frame_coverage": processed_frames == expected_frames,
|
||||
"zero_failed_frames": failed_frames == 0,
|
||||
"core_capacity_fps": (1000.0 / core["mean"])
|
||||
>= float(acceptance["minimum_core_capacity_fps"]),
|
||||
"core_path_p95_ms": core["p95"]
|
||||
<= float(acceptance["maximum_core_path_p95_ms"]),
|
||||
"inference_request_p95_ms": inference["p95"]
|
||||
<= float(acceptance["maximum_inference_request_p95_ms"]),
|
||||
"single_pass_full_raw_fov": True,
|
||||
}
|
||||
runtime: dict[str, Any] = {
|
||||
"schema_version": RUNTIME_SCHEMA,
|
||||
"status": "completed" if all(checks.values()) else "completed-gate-failed",
|
||||
"worker_host": platform.node(),
|
||||
"gpu_name": subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
).stdout.strip().splitlines()[0],
|
||||
"started_at_utc": run_started_utc.isoformat(timespec="milliseconds").replace(
|
||||
"+00:00", "Z"
|
||||
),
|
||||
"completed_at_utc": run_completed_utc.isoformat(
|
||||
timespec="milliseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
"source": {
|
||||
"video_sha256": source_sha,
|
||||
"frame_rate": frame_rate,
|
||||
"resolution": [800, 600],
|
||||
"decoded_frame_count": processed_frames,
|
||||
"valid_fov_mask_sha256": mask_sha,
|
||||
"preprocessing": "raw KB4 valid-FOV fill plus top-left letterbox; no crop/dewarp/tile",
|
||||
},
|
||||
"model": {
|
||||
"id": profile["detector"]["id"],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
"config_sha256": profile["detector"]["config_sha256"],
|
||||
"runtime": profile["detector"]["runtime"],
|
||||
"inference_requests": processed_frames,
|
||||
},
|
||||
"profile_sha256": sha256(profile_path),
|
||||
"metrics": {
|
||||
"processed_frame_count": processed_frames,
|
||||
"failed_frame_count": failed_frames,
|
||||
"wall_seconds_including_overlay_export": round(wall_seconds, 6),
|
||||
"export_throughput_fps": round(processed_frames / wall_seconds, 6),
|
||||
"core_capacity_fps": round(1000.0 / core["mean"], 6),
|
||||
"latency_ms": {key: distribution(values) for key, values in latencies.items()},
|
||||
"detection_observation_count": sum(class_counts.values()),
|
||||
"class_observation_counts": dict(sorted(class_counts.items())),
|
||||
"mean_detections_per_frame": round(
|
||||
sum(class_counts.values()) / max(processed_frames, 1), 6
|
||||
),
|
||||
"maximum_detections_per_frame": maximum_detections,
|
||||
"zero_detection_frame_count": zero_detection_frames,
|
||||
"zero_detection_frame_fraction": round(
|
||||
zero_detection_frames / max(processed_frames, 1), 9
|
||||
),
|
||||
"rejected_counts": dict(sorted(rejected_counts.items())),
|
||||
"gpu": telemetry_summary(telemetry.samples),
|
||||
},
|
||||
"acceptance": {
|
||||
"thresholds": acceptance,
|
||||
"checks": checks,
|
||||
"passed": all(checks.values()),
|
||||
},
|
||||
"artifacts": {
|
||||
"frames": {
|
||||
"file": frame_path.name,
|
||||
"byte_length": frame_path.stat().st_size,
|
||||
"sha256": sha256(frame_path),
|
||||
},
|
||||
},
|
||||
"authority": profile["authority"],
|
||||
}
|
||||
if overlay_path.is_file():
|
||||
runtime["artifacts"]["overlay_intermediate"] = {
|
||||
"file": overlay_path.name,
|
||||
"byte_length": overlay_path.stat().st_size,
|
||||
"sha256": sha256(overlay_path),
|
||||
"codec": "MPEG-4 Part 2",
|
||||
"frame_rate": frame_rate,
|
||||
}
|
||||
runtime_path = output / "runtime.json"
|
||||
runtime_path.write_text(
|
||||
json.dumps(runtime, ensure_ascii=False, indent=2, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return runtime
|
||||
|
||||
|
||||
def main() -> int:
|
||||
runtime = run(arguments())
|
||||
print(json.dumps(runtime, ensure_ascii=False, indent=2, allow_nan=False))
|
||||
return 0 if runtime["acceptance"]["passed"] else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the canonical YOLOX detector on calibration-derived K1 perspective views.
|
||||
|
||||
This module is an adapter between the already qualified K1 KB4 rectification
|
||||
from LAB E3 and the already qualified YOLOX/ByteTrack detector from LAB E8.
|
||||
It deliberately does not introduce another detector, tracker or coordinate
|
||||
system. Detections are projected back into the immutable right-camera frame
|
||||
before class-aware NMS and tracking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import concurrent.futures
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image, ImageDraw
|
||||
from run_e3_rectified_segmentation import _clahe, _rectification_maps
|
||||
from run_e5_instance_tracking import (
|
||||
_detections,
|
||||
_infer,
|
||||
_load_valid_fov,
|
||||
_nms,
|
||||
_preprocess,
|
||||
_valid_fraction,
|
||||
)
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--job", type=Path, required=True)
|
||||
parser.add_argument("--frames", type=Path, required=True)
|
||||
parser.add_argument("--rectification-profile", type=Path, required=True)
|
||||
parser.add_argument("--detector-profile", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
parser.add_argument("--triton-url", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--contrast", choices=("none", "clahe"), default="none")
|
||||
parser.add_argument(
|
||||
"--active-tiles",
|
||||
default="front,left,right,up,down",
|
||||
help="Comma-separated E3 rectification tile names evaluated this pass",
|
||||
)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--tile-size",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Detector raster size; 0 preserves the E3 profile raster",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.resolve(strict=True).read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError(f"JSON root is not an object: {path}")
|
||||
return value
|
||||
|
||||
|
||||
def _percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = (len(ordered) - 1) * percentile
|
||||
lower = math.floor(index)
|
||||
upper = math.ceil(index)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
fraction = index - lower
|
||||
return ordered[lower] * (1.0 - fraction) + ordered[upper] * fraction
|
||||
|
||||
|
||||
def _project_tile_box(
|
||||
box: list[float],
|
||||
tile: dict[str, Any],
|
||||
selected_tile: np.ndarray,
|
||||
tile_index: int,
|
||||
) -> tuple[np.ndarray, tuple[float, float]] | None:
|
||||
"""Project a perspective bbox perimeter into the raw fisheye frame."""
|
||||
|
||||
map_x = tile["raw_map_x"]
|
||||
map_y = tile["raw_map_y"]
|
||||
height, width = map_x.shape
|
||||
x1, y1, x2, y2 = (float(value) for value in box)
|
||||
x1 = float(np.clip(x1, 0.0, width - 1.0))
|
||||
x2 = float(np.clip(x2, 0.0, width - 1.0))
|
||||
y1 = float(np.clip(y1, 0.0, height - 1.0))
|
||||
y2 = float(np.clip(y2, 0.0, height - 1.0))
|
||||
if x2 <= x1 or y2 <= y1:
|
||||
return None
|
||||
|
||||
samples = max(16, int(max(x2 - x1, y2 - y1) / 4.0))
|
||||
horizontal = np.linspace(x1, x2, samples)
|
||||
vertical = np.linspace(y1, y2, samples)
|
||||
sample_x = np.concatenate(
|
||||
(horizontal, horizontal, np.full(samples, x1), np.full(samples, x2))
|
||||
)
|
||||
sample_y = np.concatenate(
|
||||
(np.full(samples, y1), np.full(samples, y2), vertical, vertical)
|
||||
)
|
||||
integer_x = np.clip(np.rint(sample_x).astype(np.int64), 0, width - 1)
|
||||
integer_y = np.clip(np.rint(sample_y).astype(np.int64), 0, height - 1)
|
||||
raw_x = map_x[integer_y, integer_x]
|
||||
raw_y = map_y[integer_y, integer_x]
|
||||
finite = np.isfinite(raw_x) & np.isfinite(raw_y)
|
||||
if not np.any(finite):
|
||||
return None
|
||||
|
||||
center_x = int(np.clip(round((x1 + x2) / 2.0), 0, width - 1))
|
||||
center_y = int(np.clip(round((y1 + y2) / 2.0), 0, height - 1))
|
||||
raw_center_x = float(map_x[center_y, center_x])
|
||||
raw_center_y = float(map_y[center_y, center_x])
|
||||
raw_height, raw_width = selected_tile.shape
|
||||
if not (
|
||||
math.isfinite(raw_center_x)
|
||||
and math.isfinite(raw_center_y)
|
||||
and 0.0 <= raw_center_x < raw_width
|
||||
and 0.0 <= raw_center_y < raw_height
|
||||
):
|
||||
return None
|
||||
owner_x = int(np.clip(round(raw_center_x), 0, raw_width - 1))
|
||||
owner_y = int(np.clip(round(raw_center_y), 0, raw_height - 1))
|
||||
if int(selected_tile[owner_y, owner_x]) != tile_index:
|
||||
return None
|
||||
|
||||
projected = np.asarray(
|
||||
[
|
||||
float(np.min(raw_x[finite])),
|
||||
float(np.min(raw_y[finite])),
|
||||
float(np.max(raw_x[finite])),
|
||||
float(np.max(raw_y[finite])),
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
projected[[0, 2]] = np.clip(projected[[0, 2]], 0.0, raw_width)
|
||||
projected[[1, 3]] = np.clip(projected[[1, 3]], 0.0, raw_height)
|
||||
return projected, (raw_center_x, raw_center_y)
|
||||
|
||||
|
||||
def _merge_raw_detections(
|
||||
candidates: list[dict[str, Any]],
|
||||
detector_profile: dict[str, Any],
|
||||
valid_mask: np.ndarray,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
detection = detector_profile["detection"]
|
||||
integral = np.pad(valid_mask.astype(np.int64), ((1, 0), (1, 0))).cumsum(0).cumsum(1)
|
||||
raw_height, raw_width = valid_mask.shape
|
||||
admitted: list[dict[str, Any]] = []
|
||||
rejected = Counter()
|
||||
for candidate in candidates:
|
||||
box = np.asarray(candidate["bbox_xyxy"], dtype=np.float64)
|
||||
valid_fraction, center_inside, area = _valid_fraction(box, integral)
|
||||
if area < float(detection["minimum_box_area_pixels"]):
|
||||
rejected["small_raw_box"] += 1
|
||||
continue
|
||||
if area / float(raw_width * raw_height) > float(
|
||||
detection["maximum_box_area_fraction"]
|
||||
):
|
||||
rejected["large_raw_box"] += 1
|
||||
continue
|
||||
if valid_fraction < float(detection["minimum_valid_fov_fraction"]):
|
||||
rejected["outside_raw_valid_fov"] += 1
|
||||
continue
|
||||
if detection["require_center_inside_valid_fov"] and not center_inside:
|
||||
rejected["raw_center_outside_valid_fov"] += 1
|
||||
continue
|
||||
enriched = dict(candidate)
|
||||
enriched["valid_fov_fraction"] = round(valid_fraction, 6)
|
||||
admitted.append(enriched)
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
class_ids = sorted({int(item["class_id"]) for item in admitted})
|
||||
for class_id in class_ids:
|
||||
group = [item for item in admitted if int(item["class_id"]) == class_id]
|
||||
boxes = np.asarray([item["bbox_xyxy"] for item in group], dtype=np.float64)
|
||||
scores = np.asarray([item["score"] for item in group], dtype=np.float64)
|
||||
keep = _nms(
|
||||
boxes,
|
||||
scores,
|
||||
float(detection["nms_iou_threshold"]),
|
||||
float(detection["nms_containment_threshold"]),
|
||||
)
|
||||
result.extend(group[index] for index in keep)
|
||||
rejected["cross_tile_duplicate"] += len(group) - len(keep)
|
||||
result.sort(key=lambda item: (-float(item["score"]), int(item["class_id"])))
|
||||
return result, dict(rejected)
|
||||
|
||||
|
||||
def detect_rectified(
|
||||
image: np.ndarray,
|
||||
*,
|
||||
maps: dict[str, Any],
|
||||
rectification_profile: dict[str, Any],
|
||||
detector_profile: dict[str, Any],
|
||||
valid_mask: np.ndarray,
|
||||
triton_url: str,
|
||||
contrast: str,
|
||||
active_tiles: set[str],
|
||||
cv2: Any,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
candidates: list[dict[str, Any]] = []
|
||||
tile_metrics: list[dict[str, Any]] = []
|
||||
tile_valid_mask = np.ones(
|
||||
(
|
||||
int(rectification_profile["rectification"]["tile_size"]),
|
||||
int(rectification_profile["rectification"]["tile_size"]),
|
||||
),
|
||||
dtype=bool,
|
||||
)
|
||||
|
||||
active_entries = [
|
||||
(tile_index, tile)
|
||||
for tile_index, tile in enumerate(maps["tiles"])
|
||||
if tile["name"] in active_tiles
|
||||
]
|
||||
|
||||
def prepare(entry: tuple[int, dict[str, Any]]) -> dict[str, Any]:
|
||||
tile_index, tile = entry
|
||||
started = time.perf_counter()
|
||||
tile_image = cv2.remap(
|
||||
image,
|
||||
tile["raw_map_x"],
|
||||
tile["raw_map_y"],
|
||||
interpolation=cv2.INTER_LINEAR,
|
||||
borderMode=cv2.BORDER_CONSTANT,
|
||||
borderValue=(114, 114, 114),
|
||||
)
|
||||
if contrast == "clahe":
|
||||
tile_image = _clahe(tile_image, rectification_profile, cv2)
|
||||
preprocess_started = time.perf_counter()
|
||||
tensor = _preprocess(tile_image, tile_valid_mask, detector_profile)
|
||||
preprocess_ms = (time.perf_counter() - preprocess_started) * 1000.0
|
||||
return {
|
||||
"tile_index": tile_index,
|
||||
"tile": tile,
|
||||
"tensor": tensor,
|
||||
"preprocess_ms": preprocess_ms,
|
||||
"preparation_ms": (time.perf_counter() - started) * 1000.0,
|
||||
}
|
||||
|
||||
# Rectification/remap is CPU work and independent for each calibrated view.
|
||||
# Keep the pool bounded to the three operational core views; Triton requests
|
||||
# below remain sequential against the single canonical model instance.
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=len(active_entries), thread_name_prefix="k1-kb4-view"
|
||||
) as executor:
|
||||
prepared = list(executor.map(prepare, active_entries))
|
||||
|
||||
for item in prepared:
|
||||
tile_index = int(item["tile_index"])
|
||||
tile = item["tile"]
|
||||
tile_started = time.perf_counter()
|
||||
output, inference_ms = _infer(
|
||||
triton_url, detector_profile["model"], item["tensor"]
|
||||
)
|
||||
tile_detections, tile_rejected = _detections(
|
||||
output, detector_profile, tile_valid_mask
|
||||
)
|
||||
owned = 0
|
||||
for detection in tile_detections:
|
||||
projected = _project_tile_box(
|
||||
detection["bbox_xyxy"], tile, maps["selected_tile"], tile_index
|
||||
)
|
||||
if projected is None:
|
||||
continue
|
||||
raw_box, raw_center = projected
|
||||
candidate = dict(detection)
|
||||
candidate["bbox_xyxy"] = [round(float(value), 6) for value in raw_box]
|
||||
candidate["raw_center_xy"] = [round(value, 6) for value in raw_center]
|
||||
candidate["rectification_tile"] = tile["name"]
|
||||
candidates.append(candidate)
|
||||
owned += 1
|
||||
tile_metrics.append(
|
||||
{
|
||||
"tile": tile["name"],
|
||||
"detections": len(tile_detections),
|
||||
"owned_detections": owned,
|
||||
"inference_ms": round(inference_ms, 6),
|
||||
"preprocess_ms": round(float(item["preprocess_ms"]), 6),
|
||||
"preparation_ms": round(float(item["preparation_ms"]), 6),
|
||||
"serial_inference_postprocess_ms": round(
|
||||
(time.perf_counter() - tile_started) * 1000.0, 6
|
||||
),
|
||||
"rejected": tile_rejected,
|
||||
}
|
||||
)
|
||||
merged, rejected = _merge_raw_detections(candidates, detector_profile, valid_mask)
|
||||
return merged, {
|
||||
"tiles": tile_metrics,
|
||||
"candidate_count": len(candidates),
|
||||
"rejected": rejected,
|
||||
}
|
||||
|
||||
|
||||
def _draw_overlay(image: np.ndarray, detections: list[dict[str, Any]]) -> Image.Image:
|
||||
output = Image.fromarray(image)
|
||||
draw = ImageDraw.Draw(output)
|
||||
palette = {
|
||||
"person": "#8cff5d",
|
||||
"bicycle": "#50d7ff",
|
||||
"car": "#ffffff",
|
||||
"motorcycle": "#ffcd57",
|
||||
"bus": "#ff8b5d",
|
||||
"truck": "#ff8b5d",
|
||||
}
|
||||
for detection in detections:
|
||||
box = tuple(float(value) for value in detection["bbox_xyxy"])
|
||||
color = palette.get(str(detection["label"]), "#ffffff")
|
||||
draw.rectangle(box, outline=color, width=3)
|
||||
label = (
|
||||
f"{detection['label']} {float(detection['score']):.0%} "
|
||||
f"[{detection['rectification_tile']}]"
|
||||
)
|
||||
text_box = draw.textbbox((box[0], max(0.0, box[1] - 18.0)), label)
|
||||
draw.rectangle(text_box, fill="#0b0b0d")
|
||||
draw.text((box[0], max(0.0, box[1] - 18.0)), label, fill=color)
|
||||
return output
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = _arguments()
|
||||
if arguments.limit < 0:
|
||||
raise RuntimeError("--limit cannot be negative")
|
||||
if arguments.tile_size and not 320 <= arguments.tile_size <= 1024:
|
||||
raise RuntimeError("--tile-size must be zero or within [320, 1024]")
|
||||
import cv2
|
||||
|
||||
rectification_profile = _read_object(arguments.rectification_profile)
|
||||
if arguments.tile_size:
|
||||
rectification_profile = copy.deepcopy(rectification_profile)
|
||||
rectification_profile["rectification"]["tile_size"] = arguments.tile_size
|
||||
detector_profile = _read_object(arguments.detector_profile)
|
||||
job = _read_object(arguments.job)
|
||||
active_tiles = {
|
||||
value.strip() for value in arguments.active_tiles.split(",") if value.strip()
|
||||
}
|
||||
configured_tiles = {
|
||||
str(tile["name"])
|
||||
for tile in rectification_profile["rectification"]["tiles"]
|
||||
}
|
||||
if not active_tiles or not active_tiles <= configured_tiles:
|
||||
raise RuntimeError("--active-tiles contains an unknown or empty tile set")
|
||||
expected_resolution = tuple(rectification_profile["source"]["resolution"])
|
||||
if expected_resolution != tuple(detector_profile["source"]["resolution"]):
|
||||
raise RuntimeError("Rectification and detector source resolutions differ")
|
||||
if (
|
||||
rectification_profile["source"]["calibration_sha256"]
|
||||
!= detector_profile["source"]["calibration_sha256"]
|
||||
):
|
||||
raise RuntimeError("Rectification and detector calibration identities differ")
|
||||
|
||||
valid_mask, _mask_metadata = _load_valid_fov(
|
||||
arguments.valid_fov_root, job, detector_profile
|
||||
)
|
||||
maps_started = time.perf_counter()
|
||||
maps = _rectification_maps(rectification_profile, valid_mask)
|
||||
maps_ms = (time.perf_counter() - maps_started) * 1000.0
|
||||
frame_paths = sorted(arguments.frames.glob("*.jpg"))
|
||||
if arguments.limit:
|
||||
frame_paths = frame_paths[: arguments.limit]
|
||||
if not frame_paths:
|
||||
raise RuntimeError("No JPEG benchmark frames were found")
|
||||
|
||||
arguments.output.mkdir(parents=True, exist_ok=False)
|
||||
overlays = arguments.output / "overlays"
|
||||
overlays.mkdir()
|
||||
documents: list[dict[str, Any]] = []
|
||||
frame_latencies: list[float] = []
|
||||
inference_latencies: list[float] = []
|
||||
detection_counts: list[int] = []
|
||||
for frame_path in frame_paths:
|
||||
image = np.asarray(Image.open(frame_path).convert("RGB"), dtype=np.uint8)
|
||||
if (image.shape[1], image.shape[0]) != expected_resolution:
|
||||
raise RuntimeError(f"Unexpected camera resolution: {frame_path}")
|
||||
started = time.perf_counter()
|
||||
detections, diagnostics = detect_rectified(
|
||||
image,
|
||||
maps=maps,
|
||||
rectification_profile=rectification_profile,
|
||||
detector_profile=detector_profile,
|
||||
valid_mask=valid_mask,
|
||||
triton_url=arguments.triton_url,
|
||||
contrast=arguments.contrast,
|
||||
active_tiles=active_tiles,
|
||||
cv2=cv2,
|
||||
)
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000.0
|
||||
frame_latencies.append(elapsed_ms)
|
||||
inference_latencies.extend(
|
||||
float(tile["inference_ms"]) for tile in diagnostics["tiles"]
|
||||
)
|
||||
detection_counts.append(len(detections))
|
||||
overlay_name = frame_path.name
|
||||
_draw_overlay(image, detections).save(
|
||||
overlays / overlay_name, format="JPEG", quality=92, optimize=True
|
||||
)
|
||||
documents.append(
|
||||
{
|
||||
"frame": frame_path.name,
|
||||
"elapsed_ms": round(elapsed_ms, 6),
|
||||
"detections": detections,
|
||||
"diagnostics": diagnostics,
|
||||
"overlay": f"overlays/{overlay_name}",
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"FRAME={frame_path.name} DETECTIONS={len(detections)} "
|
||||
f"ELAPSED_MS={elapsed_ms:.3f}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
summary = {
|
||||
"schema_version": "missioncore.rectified-yolox-benchmark/v1",
|
||||
"pipeline": "k1-kb4-cubemap5-yolox-raw-frame/v1",
|
||||
"contrast": arguments.contrast,
|
||||
"active_tiles": sorted(active_tiles),
|
||||
"tile_size": int(rectification_profile["rectification"]["tile_size"]),
|
||||
"frame_count": len(documents),
|
||||
"rectification_map_build_ms": round(maps_ms, 6),
|
||||
"latency_ms": {
|
||||
"mean": round(statistics.fmean(frame_latencies), 6),
|
||||
"p50": round(_percentile(frame_latencies, 0.5), 6),
|
||||
"p95": round(_percentile(frame_latencies, 0.95), 6),
|
||||
"maximum": round(max(frame_latencies), 6),
|
||||
},
|
||||
"tile_inference_ms": {
|
||||
"mean": round(statistics.fmean(inference_latencies), 6),
|
||||
"p95": round(_percentile(inference_latencies, 0.95), 6),
|
||||
"maximum": round(max(inference_latencies), 6),
|
||||
},
|
||||
"detections_per_frame": {
|
||||
"mean": round(statistics.fmean(detection_counts), 6),
|
||||
"minimum": min(detection_counts),
|
||||
"maximum": max(detection_counts),
|
||||
"total": sum(detection_counts),
|
||||
},
|
||||
"coverage": maps["coverage"],
|
||||
"frames": documents,
|
||||
}
|
||||
(arguments.output / "benchmark.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(json.dumps({key: value for key, value in summary.items() if key != "frames"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,326 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Qualify the rectified YOLOX + existing ByteTrack path on full RAVNOVES00."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from run_e3_rectified_segmentation import _rectification_maps
|
||||
from run_e5_instance_tracking import (
|
||||
TwoStageTracker,
|
||||
_duplicate_pairs,
|
||||
_load_valid_fov,
|
||||
_track_document,
|
||||
)
|
||||
from run_rectified_yolox_detector import (
|
||||
_draw_overlay,
|
||||
_percentile,
|
||||
_read_object,
|
||||
detect_rectified,
|
||||
)
|
||||
|
||||
|
||||
def _arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--job", type=Path, required=True)
|
||||
parser.add_argument("--video", type=Path, required=True)
|
||||
parser.add_argument("--rectification-profile", type=Path, required=True)
|
||||
parser.add_argument("--detector-profile", type=Path, required=True)
|
||||
parser.add_argument("--valid-fov-root", type=Path, required=True)
|
||||
parser.add_argument("--triton-url", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--active-tiles", default="front,left,right")
|
||||
parser.add_argument("--tile-size", type=int, default=640)
|
||||
parser.add_argument("--preview-stride", type=int, default=250)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _latency(values: list[float]) -> dict[str, float]:
|
||||
return {
|
||||
"mean": round(statistics.fmean(values), 6),
|
||||
"p50": round(_percentile(values, 0.5), 6),
|
||||
"p95": round(_percentile(values, 0.95), 6),
|
||||
"p99": round(_percentile(values, 0.99), 6),
|
||||
"maximum": round(max(values), 6),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
arguments = _arguments()
|
||||
if arguments.tile_size != 640:
|
||||
raise RuntimeError("Operational qualification is fixed to detector-native 640 tiles")
|
||||
if arguments.preview_stride < 1 or arguments.limit < 0:
|
||||
raise RuntimeError("Preview stride or frame limit is invalid")
|
||||
if not arguments.triton_url.startswith("http://"):
|
||||
raise RuntimeError("Triton URL must use the internal HTTP endpoint")
|
||||
import cv2
|
||||
|
||||
job = _read_object(arguments.job)
|
||||
if (
|
||||
job.get("job_id") != "recorded-camera-602ac89026ed12978619801d"
|
||||
or job.get("input", {}).get("session_id") != "20260720T065719Z_viewer_live"
|
||||
or job.get("input", {}).get("source_id") != "sensor.camera.right"
|
||||
or job.get("input", {}).get("segment_count") != 4489
|
||||
):
|
||||
raise RuntimeError("Qualification is not bound to immutable RAVNOVES00")
|
||||
|
||||
rectification_profile = copy.deepcopy(_read_object(arguments.rectification_profile))
|
||||
detector_profile = _read_object(arguments.detector_profile)
|
||||
rectification_profile["rectification"]["tile_size"] = arguments.tile_size
|
||||
if (
|
||||
rectification_profile["source"]["calibration_sha256"]
|
||||
!= detector_profile["source"]["calibration_sha256"]
|
||||
):
|
||||
raise RuntimeError("Calibration identity differs between detector stages")
|
||||
active_tiles = {
|
||||
value.strip() for value in arguments.active_tiles.split(",") if value.strip()
|
||||
}
|
||||
if active_tiles != {"front", "left", "right"}:
|
||||
raise RuntimeError("Operational qualification requires front,left,right views")
|
||||
|
||||
valid_mask, valid_fov = _load_valid_fov(
|
||||
arguments.valid_fov_root, job, detector_profile
|
||||
)
|
||||
maps_started = time.perf_counter()
|
||||
maps = _rectification_maps(rectification_profile, valid_mask)
|
||||
maps_ms = (time.perf_counter() - maps_started) * 1000.0
|
||||
|
||||
video = arguments.video.resolve(strict=True)
|
||||
capture = cv2.VideoCapture(str(video))
|
||||
if not capture.isOpened():
|
||||
raise RuntimeError("OpenCV could not open the immutable camera replay")
|
||||
width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
source_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
|
||||
if (width, height, source_count) != (800, 600, 4489):
|
||||
raise RuntimeError("Camera replay metadata violates the RAVNOVES00 contract")
|
||||
|
||||
output = arguments.output.resolve()
|
||||
if output.exists():
|
||||
raise RuntimeError("Qualification output must be absent")
|
||||
output.mkdir(parents=True, mode=0o700)
|
||||
previews = output / "previews"
|
||||
previews.mkdir()
|
||||
frames_path = output / "frames.jsonl"
|
||||
tracker = TwoStageTracker(detector_profile["tracking"])
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
linear_sum_assignment(np.zeros((1, 1), dtype=np.float64))
|
||||
|
||||
latency: list[float] = []
|
||||
decode_latency: list[float] = []
|
||||
detector_latency: list[float] = []
|
||||
tracking_latency: list[float] = []
|
||||
detections_by_label: Counter[str] = Counter()
|
||||
tracks_by_label: Counter[str] = Counter()
|
||||
rejections: Counter[str] = Counter()
|
||||
unique_tracks: set[int] = set()
|
||||
duplicate_pairs = 0
|
||||
processed = 0
|
||||
failures = 0
|
||||
|
||||
# One explicit warmup makes the source-rate metrics independent from model
|
||||
# initialization while preserving the first source frame for the real run.
|
||||
ok, warm_bgr = capture.read()
|
||||
if not ok:
|
||||
raise RuntimeError("Camera replay has no warmup frame")
|
||||
warm_rgb = cv2.cvtColor(warm_bgr, cv2.COLOR_BGR2RGB)
|
||||
detect_rectified(
|
||||
warm_rgb,
|
||||
maps=maps,
|
||||
rectification_profile=rectification_profile,
|
||||
detector_profile=detector_profile,
|
||||
valid_mask=valid_mask,
|
||||
triton_url=arguments.triton_url,
|
||||
contrast="none",
|
||||
active_tiles=active_tiles,
|
||||
cv2=cv2,
|
||||
)
|
||||
capture.set(cv2.CAP_PROP_POS_FRAMES, 0)
|
||||
|
||||
run_started = time.perf_counter()
|
||||
try:
|
||||
with frames_path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
while True:
|
||||
if arguments.limit and processed >= arguments.limit:
|
||||
break
|
||||
decode_started = time.perf_counter()
|
||||
ok, bgr = capture.read()
|
||||
decoded = time.perf_counter()
|
||||
if not ok:
|
||||
break
|
||||
image = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
|
||||
frame_started = time.perf_counter()
|
||||
try:
|
||||
detections, diagnostics = detect_rectified(
|
||||
image,
|
||||
maps=maps,
|
||||
rectification_profile=rectification_profile,
|
||||
detector_profile=detector_profile,
|
||||
valid_mask=valid_mask,
|
||||
triton_url=arguments.triton_url,
|
||||
contrast="none",
|
||||
active_tiles=active_tiles,
|
||||
cv2=cv2,
|
||||
)
|
||||
detected = time.perf_counter()
|
||||
tracks = tracker.update(detections, processed)
|
||||
tracked = time.perf_counter()
|
||||
except Exception:
|
||||
failures += 1
|
||||
raise
|
||||
|
||||
frame_ms = (tracked - frame_started) * 1000.0
|
||||
latency.append(frame_ms)
|
||||
decode_latency.append((decoded - decode_started) * 1000.0)
|
||||
detector_latency.append((detected - frame_started) * 1000.0)
|
||||
tracking_latency.append((tracked - detected) * 1000.0)
|
||||
detections_by_label.update(str(item["label"]) for item in detections)
|
||||
tracks_by_label.update(track.label for track in tracks)
|
||||
for track in tracks:
|
||||
unique_tracks.add(track.track_id)
|
||||
duplicate_pairs += _duplicate_pairs(tracks)
|
||||
rejections.update(diagnostics["rejected"])
|
||||
|
||||
document = {
|
||||
"schema_version": "missioncore.rectified-yolox-frame/v1",
|
||||
"frame_index": processed,
|
||||
"sequence": processed + 1,
|
||||
"detections": detections,
|
||||
"tracks": [_track_document(track) for track in tracks],
|
||||
"processing_ms": round(frame_ms, 6),
|
||||
"tile_inference_ms": round(
|
||||
sum(float(tile["inference_ms"]) for tile in diagnostics["tiles"]),
|
||||
6,
|
||||
),
|
||||
}
|
||||
stream.write(
|
||||
json.dumps(
|
||||
document,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
if processed % arguments.preview_stride == 0:
|
||||
_draw_overlay(image, detections).save(
|
||||
previews / f"frame-{processed:06d}.jpg",
|
||||
format="JPEG",
|
||||
quality=92,
|
||||
optimize=True,
|
||||
)
|
||||
processed += 1
|
||||
if processed % 100 == 0:
|
||||
stream.flush()
|
||||
print(
|
||||
f"PHASE=rectified-yolox FRAMES={processed} "
|
||||
f"LAST_MS={frame_ms:.3f}",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
capture.release()
|
||||
wall_seconds = time.perf_counter() - run_started
|
||||
expected = arguments.limit or source_count
|
||||
steady = latency[1:] if len(latency) > 1 else latency
|
||||
effective_fps = processed / wall_seconds
|
||||
steady_latency = _latency(steady)
|
||||
checks = {
|
||||
"complete_frame_accounting": processed == expected,
|
||||
"zero_failures": failures == 0,
|
||||
"minimum_effective_fps_9_5": effective_fps >= 9.5,
|
||||
"maximum_steady_p95_ms_100": steady_latency["p95"] <= 100.0,
|
||||
"same_class_duplicate_pairs_zero": duplicate_pairs == 0,
|
||||
}
|
||||
accepted = all(checks.values())
|
||||
report: dict[str, Any] = {
|
||||
"schema_version": "missioncore.rectified-yolox-qualification/v1",
|
||||
"state": "accepted" if accepted else "rejected",
|
||||
"source": {
|
||||
"job_id": job["job_id"],
|
||||
"session_id": job["input"]["session_id"],
|
||||
"source_id": job["input"]["source_id"],
|
||||
"frame_count": source_count,
|
||||
"calibration_sha256": detector_profile["source"]["calibration_sha256"],
|
||||
},
|
||||
"pipeline": {
|
||||
"id": "k1-kb4-core3-yolox-bytetrack/v1",
|
||||
"rectification": "five-perspective-gnomonic/v1",
|
||||
"active_tiles": sorted(active_tiles),
|
||||
"tile_size": arguments.tile_size,
|
||||
"detector": detector_profile["model"],
|
||||
"tracker": detector_profile["tracking"],
|
||||
"peripheral_views": "up/down remain on the existing 2 Hz semantic cadence",
|
||||
},
|
||||
"runtime": {
|
||||
"hostname": platform.node(),
|
||||
"python": platform.python_version(),
|
||||
"opencv": cv2.__version__,
|
||||
"rectification_map_build_ms": round(maps_ms, 6),
|
||||
"wall_seconds": round(wall_seconds, 6),
|
||||
"effective_fps": round(effective_fps, 6),
|
||||
},
|
||||
"metrics": {
|
||||
"frames_processed": processed,
|
||||
"failures": failures,
|
||||
"latency_ms": _latency(latency),
|
||||
"steady_latency_ms": steady_latency,
|
||||
"decode_latency_ms": _latency(decode_latency),
|
||||
"detector_latency_ms": _latency(detector_latency),
|
||||
"tracking_latency_ms": _latency(tracking_latency),
|
||||
"detections": int(sum(detections_by_label.values())),
|
||||
"detections_by_label": dict(sorted(detections_by_label.items())),
|
||||
"unique_confirmed_tracks": len(unique_tracks),
|
||||
"track_observations": int(sum(tracks_by_label.values())),
|
||||
"track_observations_by_label": dict(sorted(tracks_by_label.items())),
|
||||
"rejections": dict(sorted(rejections.items())),
|
||||
"same_class_duplicate_pairs_iou_ge_0_8": duplicate_pairs,
|
||||
"tracker_tracks_created": tracker.created,
|
||||
"tracker_tracks_retired": tracker.retired,
|
||||
},
|
||||
"acceptance": {"accepted": accepted, "checks": checks},
|
||||
"valid_fov": valid_fov,
|
||||
"limitations": [
|
||||
"RAVNOVES00 has no independent exhaustive object ground truth.",
|
||||
"This gate qualifies runtime and visual evidence, not safety accuracy.",
|
||||
"LiDAR association, temporal world state and segmentation are existing downstream stages and are not recomputed by this detector-only gate.",
|
||||
],
|
||||
"artifacts": {
|
||||
"frames": "frames.jsonl",
|
||||
"previews": "previews/",
|
||||
},
|
||||
}
|
||||
(output / "qualification.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"accepted": accepted,
|
||||
"frames_processed": processed,
|
||||
"effective_fps": round(effective_fps, 6),
|
||||
"steady_p95_ms": steady_latency["p95"],
|
||||
"detections": report["metrics"]["detections"],
|
||||
"unique_confirmed_tracks": len(unique_tracks),
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
raise SystemExit(0 if accepted else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user