SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): define semantic object understanding

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 16:43:47 +03:00
parent 1084984da2
commit 6276bbf324
28 changed files with 7028 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/perception/m48s-mask-grounding-dino-evidence-v0.json
+37
View File
@@ -0,0 +1,37 @@
{
"schema_version": "missioncore.m48s-mask-grounding-dino-profile/v0",
"profile_id": "m48s-mask-grounding-dino-research-shadow/v0",
"model": {
"catalog": "NVIDIA NGC",
"catalog_url": "https://catalog.ngc.nvidia.com/orgs/nvidia/tao/models/pretrained_mask_grounding_dino_v2/mask_grounding_dino_swin_tiny_research_deployable_v2.0/version-history",
"artifact": "mask_grounding_dino_epoch_021.onnx",
"artifact_sha256": "4e8b066cf7782ae6b4269255b861c205f4b07b33ff752e5eaf297c42ecfb2f5e",
"license_scope": "research-only",
"commercial_checkpoint_acquired": false
},
"runtime": {
"worker_node": "DESKTOP-OPJ8J04",
"gpu": "NVIDIA GeForce RTX 4090",
"container_reference": "nvcr.io/nvidia/tao/tao-toolkit:7.0.1-deploy",
"container_image_id": "sha256:2a3095330dd83e4314aada21fc7d184fcdb442a9131d591e5593ca5497394e7a",
"tensorrt_precision": "FP16",
"engine_sha256": "1d127107d15f18dcddafd7345a0a766acfee204d63c5ab697eaabada2c7afcd2",
"engine_size_bytes": 407588756
},
"binding": {
"minimum_mask_iou": 0.9,
"minimum_projected_points": 4,
"minimum_support_fraction": 0.5,
"maximum_secondary_support_fraction": 0.25,
"minimum_label_confidence": 0.3,
"minimum_label_margin": 0.1
},
"authority": {
"ground_truth": false,
"independent_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
config/perception/m48s-mask-grounding-dino-shadow-v0.yaml
+71
View File
@@ -0,0 +1,71 @@
model_name: missioncore-m48s-mask-grounding-dino-shadow
results_dir: /workspace/probe/results
wandb:
enable: false
model:
backbone: swin_tiny_224_1k
num_queries: 900
num_feature_levels: 4
dec_layers: 6
enc_layers: 6
dim_feedforward: 2048
dropout_ratio: 0.0
log_scale: auto
class_embed_bias: true
num_region_queries: 100
loss_types:
- labels
- boxes
- masks
- rela
dataset:
infer_data_sources:
image_dir: /workspace/probe/input
data_type: OD
captions:
- adult person
- child
- dog
- bicycle
- kick scooter
- stroller
- shopping cart
- car
- bus
- truck
- motorcycle
- trash bin
- bollard
- post
- concrete hemisphere
- traffic cone
- road sign
- temporary barrier
batch_size: 1
workers: 1
pin_memory: true
max_labels: 80
has_mask: true
inference:
trt_engine: /workspace/model/mask_grounding_dino_swin_tiny_research_fp16.engine
results_dir: /workspace/probe/results
batch_size: -1
input_width: 960
input_height: 544
conf_threshold: 0.25
ioi_threshold: 0.5
nms_threshold: 0.2
text_threshold: 0.3
outline_width: 3
gen_trt_engine:
onnx_file: /workspace/model/mask_grounding_dino_epoch_021.onnx
trt_engine: /workspace/model/mask_grounding_dino_swin_tiny_research_fp16.engine
results_dir: /workspace/model/engine-build
batch_size: -1
verbose: false
tensorrt:
workspace_size: 8192
min_batch_size: 1
opt_batch_size: 1
max_batch_size: 1
data_type: FP16
config/perception/m48s-semantic-object-quality-v0.json
+26
View File
@@ -0,0 +1,26 @@
{
"schema_version": "missioncore.semantic-object-quality-profile/v0",
"profile_id": "m48s-urban-semantic-object-quality/v0",
"vocabulary_id": "missioncore.urban-object-semantics/v0",
"top_k": 5,
"thresholds": {
"minimum_prediction_coverage": 1.0,
"minimum_exact_top1_accuracy": 0.8,
"minimum_coarse_group_accuracy": 0.95,
"minimum_exact_top_k_recall": 0.95,
"maximum_unresolved_fraction": 0.1,
"maximum_conflict_fraction": 0.05
},
"scope": {
"object_presence_scored": false,
"semantic_class_scored": true,
"risk_policy_scored": false,
"requires_separate_adjudicated_semantic_truth": true
},
"authority": {
"physical_live": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}
config/perception/object-semantic-vocabulary-v0.json
+233
View File
@@ -0,0 +1,233 @@
{
"schema_version": "missioncore.object-semantic-vocabulary/v0",
"vocabulary_id": "missioncore.urban-object-semantics/v0",
"status": "experimental",
"scope": "urban-ground-robot",
"classes": [
{
"class_id": "object.unknown",
"parent_id": null,
"aliases": ["unknown", "unknown_object"],
"agency_prior": "unknown",
"risk_traits": ["unknown-clearance", "unknown-motion-potential"]
},
{
"class_id": "human.unknown",
"parent_id": "object.unknown",
"aliases": ["person", "human", "pedestrian"],
"agency_prior": "animate",
"risk_traits": ["vulnerable-road-user", "unpredictable-motion"]
},
{
"class_id": "human.adult",
"parent_id": "human.unknown",
"aliases": ["adult", "adult_person"],
"agency_prior": "animate",
"risk_traits": ["vulnerable-road-user", "unpredictable-motion"]
},
{
"class_id": "human.child",
"parent_id": "human.unknown",
"aliases": ["child", "kid", "toddler"],
"agency_prior": "animate",
"risk_traits": ["vulnerable-road-user", "high-unpredictability"]
},
{
"class_id": "animal.unknown",
"parent_id": "object.unknown",
"aliases": ["animal", "unknown_animal"],
"agency_prior": "animate",
"risk_traits": ["unpredictable-motion"]
},
{
"class_id": "animal.dog",
"parent_id": "animal.unknown",
"aliases": ["dog", "canine"],
"agency_prior": "animate",
"risk_traits": ["unpredictable-motion", "close-interaction-risk"]
},
{
"class_id": "vehicle.unknown",
"parent_id": "object.unknown",
"aliases": ["vehicle", "unknown_vehicle"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "large-footprint"]
},
{
"class_id": "vehicle.car",
"parent_id": "vehicle.unknown",
"aliases": ["car", "automobile"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "reversing-risk"]
},
{
"class_id": "vehicle.heavy",
"parent_id": "vehicle.unknown",
"aliases": ["heavy_vehicle", "truck", "bus"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "large-footprint", "blind-zone"]
},
{
"class_id": "vehicle.bicycle",
"parent_id": "vehicle.unknown",
"aliases": ["bicycle", "bike"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "vulnerable-road-user"]
},
{
"class_id": "vehicle.motorcycle",
"parent_id": "vehicle.unknown",
"aliases": ["motorcycle", "motorbike"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "rapid-motion"]
},
{
"class_id": "vehicle.kick-scooter",
"parent_id": "vehicle.unknown",
"aliases": ["kick_scooter", "scooter"],
"agency_prior": "self-propelled",
"risk_traits": ["may-start-moving", "vulnerable-road-user"]
},
{
"class_id": "mobility.unknown",
"parent_id": "object.unknown",
"aliases": ["passive_mobility", "wheeled_object"],
"agency_prior": "inert",
"risk_traits": ["may-be-pushed", "rolling-potential"]
},
{
"class_id": "mobility.stroller",
"parent_id": "mobility.unknown",
"aliases": ["stroller", "pram"],
"agency_prior": "inert",
"risk_traits": ["may-be-pushed", "vulnerable-occupant"]
},
{
"class_id": "mobility.cart",
"parent_id": "mobility.unknown",
"aliases": ["cart", "shopping_cart", "trolley"],
"agency_prior": "inert",
"risk_traits": ["may-be-pushed", "rolling-potential"]
},
{
"class_id": "static.unknown",
"parent_id": "object.unknown",
"aliases": ["static_obstacle", "generic_obstacle", "obstacle"],
"agency_prior": "inert",
"risk_traits": ["static-clearance"]
},
{
"class_id": "static.trash-bin",
"parent_id": "static.unknown",
"aliases": ["trash_bin", "garbage_bin", "waste_bin"],
"agency_prior": "inert",
"risk_traits": ["static-clearance", "movable-by-external-force"]
},
{
"class_id": "static.bollard",
"parent_id": "static.unknown",
"aliases": ["bollard", "road_bollard"],
"agency_prior": "inert",
"risk_traits": ["small-static", "rigid-clearance"]
},
{
"class_id": "static.post",
"parent_id": "static.unknown",
"aliases": ["post", "pole", "pipe_post"],
"agency_prior": "inert",
"risk_traits": ["small-static", "rigid-clearance"]
},
{
"class_id": "static.concrete-hemisphere",
"parent_id": "static.unknown",
"aliases": ["concrete_hemisphere", "hemisphere", "dome_barrier"],
"agency_prior": "inert",
"risk_traits": ["low-profile", "rigid-clearance"]
},
{
"class_id": "static.traffic-cone",
"parent_id": "static.unknown",
"aliases": ["traffic_cone", "cone"],
"agency_prior": "inert",
"risk_traits": ["small-static", "temporary-road-control"]
},
{
"class_id": "static.road-sign",
"parent_id": "static.unknown",
"aliases": ["road_sign", "traffic_sign"],
"agency_prior": "inert",
"risk_traits": ["static-clearance", "road-control-device"]
},
{
"class_id": "static.temporary-barrier",
"parent_id": "static.unknown",
"aliases": ["temporary_barrier", "road_barrier", "barrier"],
"agency_prior": "inert",
"risk_traits": ["static-clearance", "temporary-road-control"]
},
{
"class_id": "terrain.unknown",
"parent_id": "object.unknown",
"aliases": ["terrain_hazard", "surface_hazard"],
"agency_prior": "inert",
"risk_traits": ["surface-discontinuity"]
},
{
"class_id": "terrain.curb",
"parent_id": "terrain.unknown",
"aliases": ["curb", "sidewalk_curb"],
"agency_prior": "inert",
"risk_traits": ["surface-discontinuity", "vehicle-dependent-traversability"]
},
{
"class_id": "terrain.step",
"parent_id": "terrain.unknown",
"aliases": ["step", "stair"],
"agency_prior": "inert",
"risk_traits": ["surface-discontinuity", "vehicle-dependent-traversability"]
},
{
"class_id": "terrain.pothole",
"parent_id": "terrain.unknown",
"aliases": ["pothole", "road_hole"],
"agency_prior": "inert",
"risk_traits": ["negative-obstacle", "vehicle-dependent-traversability"]
},
{
"class_id": "terrain.drainage-grate",
"parent_id": "terrain.unknown",
"aliases": ["drainage_grate", "drain", "road_grate"],
"agency_prior": "inert",
"risk_traits": ["surface-discontinuity", "wheel-entrapment"]
},
{
"class_id": "natural.unknown",
"parent_id": "object.unknown",
"aliases": ["natural_obstacle", "debris"],
"agency_prior": "inert",
"risk_traits": ["irregular-clearance"]
},
{
"class_id": "natural.rock",
"parent_id": "natural.unknown",
"aliases": ["rock", "stone"],
"agency_prior": "inert",
"risk_traits": ["irregular-clearance", "rigid-clearance"]
},
{
"class_id": "natural.branch",
"parent_id": "natural.unknown",
"aliases": ["branch", "fallen_branch"],
"agency_prior": "inert",
"risk_traits": ["irregular-clearance", "deformable-obstacle"]
}
],
"policies": {
"occupancy_independent_of_semantics": true,
"unknown_preserves_obstacle": true,
"class_prior_is_not_observed_state": true,
"risk_is_advisory_only": true,
"planner_command_authority": false,
"max_hypotheses": 5
}
}
config/perception/open-vocabulary-semantic-shadow-v0.json
+86
View File
@@ -0,0 +1,86 @@
{
"schema_version": "missioncore.open-vocabulary-semantic-shadow-profile/v0",
"profile_id": "m48s-raw-kb4-urban-semantics-shadow/v0",
"status": "experimental-shadow",
"source": {
"session_id": "20260720T065719Z_viewer_live",
"source_id": "RAVNOVES00",
"camera_source_id": "sensor.camera.right",
"coordinate_space": "raw-kb4",
"width": 800,
"height": 600,
"decoded_frame_basis": "zero-based-decoded-video-frame",
"decoded_image_encoding": "jpeg-q2-yuvj420p",
"valid_fov_result_id": "valid-fov-mask-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2",
"valid_fov_mask_sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63",
"valid_fov_fill_value": 114
},
"provider": {
"provider_id": "nvidia-tao-grounding-dino-trt/v1",
"name": "NVIDIA TAO Grounding DINO Swin-Tiny Commercial",
"version": "1.0",
"model_id": "grounding-dino-swin-tiny-commercial/v1",
"model_file": "grounding_dino_swin_tiny_commercial_deployable.onnx",
"model_sha256": "6895acdc6b588e923f753e37b3bd18869e064256e5ecc1b2b9853e8c51125f94",
"model_revision": "nvidia-tao-7.0.1-commercial-v1.0",
"engine_file": "grounding_dino_swin_tiny_commercial_fp16.engine",
"engine_sha256": "63213f60730ce8577047ff508dfe06113269347501e2dd832aeb56c89bc11489",
"engine_precision": "FP16",
"container_reference": "nvcr.io/nvidia/tao/tao-toolkit:7.0.1-deploy",
"container_image_id": "sha256:2a3095330dd83e4314aada21fc7d184fcdb442a9131d591e5593ca5497394e7a",
"preprocess_id": "raw-kb4-valid-fov-fill114-800x600-to-960x544/v1",
"engine_input_width": 960,
"engine_input_height": 544
},
"postprocessing": {
"minimum_input_confidence": 0.25,
"minimum_box_area_fraction": 0.00002,
"maximum_box_area_fraction": 0.35,
"minimum_valid_fov_fraction": 0.5,
"require_center_inside_valid_fov": true,
"fusion_iou_threshold": 0.7,
"selected_minimum_confidence": 0.5,
"selected_minimum_margin": 0.1,
"max_hypotheses": 5
},
"prompt_groups": [
{
"prompt_set_id": "urban-static/v0",
"captions": [
"trash bin",
"shopping cart",
"bollard",
"post",
"concrete hemisphere",
"traffic cone",
"road sign",
"temporary barrier"
]
},
{
"prompt_set_id": "urban-agents/v0",
"captions": [
"adult person",
"child",
"dog",
"stroller",
"kick scooter",
"bicycle"
]
},
{
"prompt_set_id": "urban-vehicles/v0",
"captions": ["car", "truck", "bus", "motorcycle"]
}
],
"semantic_vocabulary_id": "missioncore.urban-object-semantics/v0",
"risk_policy_id": "urban-object-risk-shadow/v0",
"authority": {
"ground_truth": false,
"independent_truth": false,
"candidate_accepted": false,
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false
}
}