SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): define semantic object understanding

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-25 16:43:47 +03:00
parent 1084984da2
commit 6276bbf324
28 changed files with 7028 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/laboratory/semantic_object_quality.py
+454
View File
@@ -0,0 +1,454 @@
"""Separate semantic-class evaluation over geometry-bound object projections.
This companion contour does not mutate the class-free M4.8 truth or score
object presence. It evaluates canonical semantic resolution only after a
separate two-reviewer, adjudicated semantic label set exists.
"""
from __future__ import annotations
import json
import math
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from k1link.perception.object_understanding import (
ObjectSemanticVocabulary,
ObjectUnderstanding,
SemanticResolution,
validate_object_understanding,
)
SEMANTIC_TRUTH_LABEL_SCHEMA: Final = "missioncore.semantic-object-truth-label/v0"
SEMANTIC_QUALITY_PROFILE_SCHEMA: Final = "missioncore.semantic-object-quality-profile/v0"
SEMANTIC_QUALITY_REPORT_SCHEMA: Final = "missioncore.semantic-object-quality-report/v0"
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$")
_FALSE_AUTHORITY: Final = {
"physical_live": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
class SemanticObjectQualityError(ValueError):
"""A semantic truth, profile or evaluation input is incompatible."""
@dataclass(frozen=True, slots=True)
class SemanticTruthLabel:
"""One independently reviewed canonical class bound to existing geometry."""
label_id: str
observation_id: str
source_id: str
frame_id: str
class_id: str
reviewer_count: int
adjudicated: bool
def __post_init__(self) -> None:
for value, label in (
(self.label_id, "semantic truth label id"),
(self.observation_id, "semantic truth observation id"),
(self.source_id, "semantic truth source id"),
(self.frame_id, "semantic truth frame id"),
(self.class_id, "semantic truth class id"),
):
_identifier(value, label)
if (
not isinstance(self.reviewer_count, int)
or isinstance(self.reviewer_count, bool)
or self.reviewer_count < 2
):
raise SemanticObjectQualityError("semantic truth requires two independent reviewers")
if self.adjudicated is not True:
raise SemanticObjectQualityError("semantic truth must be adjudicated")
def to_dict(self) -> dict[str, object]:
return {
"schema_version": SEMANTIC_TRUTH_LABEL_SCHEMA,
"label_id": self.label_id,
"observation_id": self.observation_id,
"source_id": self.source_id,
"frame_id": self.frame_id,
"class_id": self.class_id,
"reviewer_count": self.reviewer_count,
"adjudicated": self.adjudicated,
}
@classmethod
def from_dict(cls, value: object) -> SemanticTruthLabel:
document = _object(value, "semantic truth label")
_exact_keys(
document,
{
"schema_version",
"label_id",
"observation_id",
"source_id",
"frame_id",
"class_id",
"reviewer_count",
"adjudicated",
},
"semantic truth label",
)
if document.get("schema_version") != SEMANTIC_TRUTH_LABEL_SCHEMA:
raise SemanticObjectQualityError("semantic truth label schema is incompatible")
return cls(
label_id=_string(document, "label_id"),
observation_id=_string(document, "observation_id"),
source_id=_string(document, "source_id"),
frame_id=_string(document, "frame_id"),
class_id=_string(document, "class_id"),
reviewer_count=_integer(document, "reviewer_count"),
adjudicated=_boolean(document, "adjudicated"),
)
@dataclass(frozen=True, slots=True)
class SemanticObjectQualityProfile:
"""Bounded evaluation policy for a frozen semantic provider candidate."""
profile_id: str
vocabulary_id: str
top_k: int
minimum_prediction_coverage: float
minimum_exact_top1_accuracy: float
minimum_coarse_group_accuracy: float
minimum_exact_top_k_recall: float
maximum_unresolved_fraction: float
maximum_conflict_fraction: float
def __post_init__(self) -> None:
_identifier(self.profile_id, "semantic quality profile id")
_identifier(self.vocabulary_id, "semantic quality vocabulary id")
if (
not isinstance(self.top_k, int)
or isinstance(self.top_k, bool)
or not 1 <= self.top_k <= 5
):
raise SemanticObjectQualityError("semantic quality top-k is invalid")
for value, label in (
(self.minimum_prediction_coverage, "minimum prediction coverage"),
(self.minimum_exact_top1_accuracy, "minimum exact top-1 accuracy"),
(self.minimum_coarse_group_accuracy, "minimum coarse group accuracy"),
(self.minimum_exact_top_k_recall, "minimum exact top-k recall"),
(self.maximum_unresolved_fraction, "maximum unresolved fraction"),
(self.maximum_conflict_fraction, "maximum conflict fraction"),
):
_fraction(value, label)
@dataclass(frozen=True, slots=True)
class SemanticQualityCase:
"""One deterministic truth/prediction comparison row."""
label_id: str
observation_id: str
truth_class_id: str
resolution: str
selected_class_id: str | None
exact_top1_correct: bool
coarse_group_correct: bool
exact_top_k_hit: bool
def to_dict(self) -> dict[str, object]:
return {
"label_id": self.label_id,
"observation_id": self.observation_id,
"truth_class_id": self.truth_class_id,
"resolution": self.resolution,
"selected_class_id": self.selected_class_id,
"exact_top1_correct": self.exact_top1_correct,
"coarse_group_correct": self.coarse_group_correct,
"exact_top_k_hit": self.exact_top_k_hit,
}
@dataclass(frozen=True, slots=True)
class SemanticObjectQualityResult:
"""In-memory deterministic semantic report and comparison ledger."""
report: dict[str, object]
cases: tuple[SemanticQualityCase, ...]
def load_semantic_object_quality_profile(path: Path) -> SemanticObjectQualityProfile:
"""Read a strict semantic evaluation profile."""
try:
document = json.loads(path.expanduser().resolve(strict=True).read_text("utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SemanticObjectQualityError("semantic quality profile cannot be read") from exc
root = _object(document, "semantic quality profile")
_exact_keys(
root,
{
"schema_version",
"profile_id",
"vocabulary_id",
"top_k",
"thresholds",
"scope",
"authority",
},
"semantic quality profile",
)
if root.get("schema_version") != SEMANTIC_QUALITY_PROFILE_SCHEMA:
raise SemanticObjectQualityError("semantic quality profile schema is incompatible")
scope = _object(root.get("scope"), "semantic quality scope")
_exact_keys(
scope,
{
"object_presence_scored",
"semantic_class_scored",
"risk_policy_scored",
"requires_separate_adjudicated_semantic_truth",
},
"semantic quality scope",
)
if scope != {
"object_presence_scored": False,
"semantic_class_scored": True,
"risk_policy_scored": False,
"requires_separate_adjudicated_semantic_truth": True,
}:
raise SemanticObjectQualityError("semantic quality scope changed")
if _object(root.get("authority"), "semantic quality authority") != _FALSE_AUTHORITY:
raise SemanticObjectQualityError("semantic quality authority changed")
thresholds = _object(root.get("thresholds"), "semantic quality thresholds")
_exact_keys(
thresholds,
{
"minimum_prediction_coverage",
"minimum_exact_top1_accuracy",
"minimum_coarse_group_accuracy",
"minimum_exact_top_k_recall",
"maximum_unresolved_fraction",
"maximum_conflict_fraction",
},
"semantic quality thresholds",
)
return SemanticObjectQualityProfile(
profile_id=_string(root, "profile_id"),
vocabulary_id=_string(root, "vocabulary_id"),
top_k=_integer(root, "top_k"),
minimum_prediction_coverage=_number(thresholds, "minimum_prediction_coverage"),
minimum_exact_top1_accuracy=_number(thresholds, "minimum_exact_top1_accuracy"),
minimum_coarse_group_accuracy=_number(thresholds, "minimum_coarse_group_accuracy"),
minimum_exact_top_k_recall=_number(thresholds, "minimum_exact_top_k_recall"),
maximum_unresolved_fraction=_number(thresholds, "maximum_unresolved_fraction"),
maximum_conflict_fraction=_number(thresholds, "maximum_conflict_fraction"),
)
def score_semantic_object_quality(
*,
predictions: tuple[ObjectUnderstanding, ...],
truth: tuple[SemanticTruthLabel, ...],
vocabulary: ObjectSemanticVocabulary,
profile: SemanticObjectQualityProfile,
) -> SemanticObjectQualityResult:
"""Score semantics on geometry-bound truth without scoring object presence."""
if not truth or any(not isinstance(item, SemanticTruthLabel) for item in truth):
raise SemanticObjectQualityError("semantic truth set is invalid")
if any(not isinstance(item, ObjectUnderstanding) for item in predictions):
raise SemanticObjectQualityError("semantic prediction set is invalid")
if profile.vocabulary_id != vocabulary.vocabulary_id:
raise SemanticObjectQualityError("semantic quality vocabulary changed")
truth_ids = tuple(item.observation_id for item in truth)
if len(set(truth_ids)) != len(truth_ids):
raise SemanticObjectQualityError("semantic truth observations must be unique")
prediction_ids = tuple(item.observation.observation_id for item in predictions)
if len(set(prediction_ids)) != len(prediction_ids):
raise SemanticObjectQualityError("semantic prediction observations must be unique")
declared_classes = {item.class_id for item in vocabulary.classes}
if any(item.class_id not in declared_classes for item in truth):
raise SemanticObjectQualityError("semantic truth uses an undeclared class")
for candidate in predictions:
validate_object_understanding(candidate, vocabulary)
by_observation = {item.observation.observation_id: item for item in predictions}
cases: list[SemanticQualityCase] = []
predicted_count = 0
selected_count = 0
conflict_count = 0
for label in truth:
prediction = by_observation.get(label.observation_id)
if prediction is None:
cases.append(
SemanticQualityCase(
label_id=label.label_id,
observation_id=label.observation_id,
truth_class_id=label.class_id,
resolution="unavailable",
selected_class_id=None,
exact_top1_correct=False,
coarse_group_correct=False,
exact_top_k_hit=False,
)
)
continue
predicted_count += 1
observation = prediction.observation
if observation.source_id != label.source_id or observation.frame_id != label.frame_id:
raise SemanticObjectQualityError("semantic prediction escaped its truth source frame")
selected = prediction.semantic.selected_class_id
if prediction.semantic.resolution is SemanticResolution.SELECTED:
selected_count += 1
if prediction.semantic.resolution is SemanticResolution.CONFLICT:
conflict_count += 1
exact = selected == label.class_id
coarse = selected is not None and _coarse_group(selected, vocabulary) == _coarse_group(
label.class_id, vocabulary
)
top_k_ids = {item.class_id for item in prediction.hypotheses[: profile.top_k]}
cases.append(
SemanticQualityCase(
label_id=label.label_id,
observation_id=label.observation_id,
truth_class_id=label.class_id,
resolution=prediction.semantic.resolution.value,
selected_class_id=selected,
exact_top1_correct=exact,
coarse_group_correct=coarse,
exact_top_k_hit=label.class_id in top_k_ids,
)
)
truth_count = len(truth)
exact_accuracy = sum(item.exact_top1_correct for item in cases) / truth_count
coarse_accuracy = sum(item.coarse_group_correct for item in cases) / truth_count
top_k_recall = sum(item.exact_top_k_hit for item in cases) / truth_count
prediction_coverage = predicted_count / truth_count
unresolved_fraction = (truth_count - selected_count) / truth_count
conflict_fraction = conflict_count / truth_count
gates = {
"prediction_coverage": (prediction_coverage >= profile.minimum_prediction_coverage),
"exact_top1_accuracy": (exact_accuracy >= profile.minimum_exact_top1_accuracy),
"coarse_group_accuracy": (coarse_accuracy >= profile.minimum_coarse_group_accuracy),
"exact_top_k_recall": (top_k_recall >= profile.minimum_exact_top_k_recall),
"unresolved_fraction": (unresolved_fraction <= profile.maximum_unresolved_fraction),
"conflict_fraction": (conflict_fraction <= profile.maximum_conflict_fraction),
}
report: dict[str, object] = {
"schema_version": SEMANTIC_QUALITY_REPORT_SCHEMA,
"profile_id": profile.profile_id,
"vocabulary_id": vocabulary.vocabulary_id,
"scope": {
"object_presence_scored": False,
"semantic_class_scored": True,
"risk_policy_scored": False,
},
"metrics": {
"truth_count": truth_count,
"prediction_count": predicted_count,
"selected_count": selected_count,
"conflict_count": conflict_count,
"prediction_coverage": prediction_coverage,
"exact_top1_accuracy": exact_accuracy,
"coarse_group_accuracy": coarse_accuracy,
"exact_top_k_recall": top_k_recall,
"unresolved_fraction": unresolved_fraction,
"conflict_fraction": conflict_fraction,
},
"gates": gates,
"candidate_semantic_gate_passed": all(gates.values()),
"limitations": [
"Object presence and geometry quality remain owned by the class-free contour.",
"Semantic qualification does not qualify risk policy or physical motion.",
"No navigation, safety, command or actuation authority is granted.",
],
"authority": dict(_FALSE_AUTHORITY),
}
return SemanticObjectQualityResult(report=report, cases=tuple(cases))
def _coarse_group(
class_id: str,
vocabulary: ObjectSemanticVocabulary,
) -> str:
ancestors = vocabulary.ancestors(class_id)
if not ancestors:
return class_id
non_root = tuple(item for item in ancestors if item != "object.unknown")
return non_root[-1] if non_root else class_id
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise SemanticObjectQualityError(f"{label} must be an object")
return value
def _exact_keys(document: dict[str, object], expected: set[str], label: str) -> None:
if set(document) != expected:
raise SemanticObjectQualityError(f"{label} fields are incompatible")
def _string(document: dict[str, object], key: str) -> str:
value = document.get(key)
if not isinstance(value, str) or not value:
raise SemanticObjectQualityError(f"{key} must be a nonempty string")
return value
def _integer(document: dict[str, object], key: str) -> int:
value = document.get(key)
if not isinstance(value, int) or isinstance(value, bool):
raise SemanticObjectQualityError(f"{key} must be an integer")
return value
def _boolean(document: dict[str, object], key: str) -> bool:
value = document.get(key)
if not isinstance(value, bool):
raise SemanticObjectQualityError(f"{key} must be boolean")
return value
def _number(document: dict[str, object], key: str) -> float:
value = document.get(key)
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
):
raise SemanticObjectQualityError(f"{key} must be finite")
return float(value)
def _fraction(value: object, label: str) -> float:
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
or not 0.0 <= float(value) <= 1.0
):
raise SemanticObjectQualityError(f"{label} must be within [0, 1]")
return float(value)
def _identifier(value: str, label: str) -> None:
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
raise SemanticObjectQualityError(f"{label} is not a safe identifier")
__all__ = [
"SEMANTIC_QUALITY_PROFILE_SCHEMA",
"SEMANTIC_QUALITY_REPORT_SCHEMA",
"SEMANTIC_TRUTH_LABEL_SCHEMA",
"SemanticObjectQualityError",
"SemanticObjectQualityProfile",
"SemanticObjectQualityResult",
"SemanticQualityCase",
"SemanticTruthLabel",
"load_semantic_object_quality_profile",
"score_semantic_object_quality",
]
src/k1link/perception/geometry_semantic_roi.py
+272
View File
@@ -0,0 +1,272 @@
"""Geometry-first raw-KB4 regions for semantic classification shadows."""
from __future__ import annotations
import math
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from PIL import Image
from .contracts import BoundingRegion2D, ObstacleObservation
from .geometry import GeometryFrame
from .geometry_math import project_map_points_kb4
from .open_vocabulary_semantics import OpenVocabularyDetection, SemanticFusionResult
class GeometrySemanticRoiError(ValueError):
"""A geometry-owned semantic ROI or crop is incompatible."""
@dataclass(frozen=True, slots=True)
class GeometrySemanticRoiProfile:
minimum_projected_points: int = 4
minimum_crop_width: int = 96
minimum_crop_height: int = 96
padding_fraction: float = 0.25
minimum_padding_pixels: int = 16
def __post_init__(self) -> None:
if (
self.minimum_projected_points < 2
or self.minimum_crop_width < 32
or self.minimum_crop_height < 32
or not math.isfinite(self.padding_fraction)
or not 0.0 <= self.padding_fraction <= 1.0
or self.minimum_padding_pixels < 0
):
raise GeometrySemanticRoiError("geometry semantic ROI profile is invalid")
@dataclass(frozen=True, slots=True)
class GeometrySemanticRoi:
roi_id: str
frame_index: int
observation: ObstacleObservation
core_region: BoundingRegion2D
crop_region: BoundingRegion2D
projected_point_count: int
crop_name: str
def to_dict(self) -> dict[str, object]:
return {
"roi_id": self.roi_id,
"frame_index": self.frame_index,
"observation": self.observation.to_dict(),
"core_region": self.core_region.to_dict(),
"crop_region": self.crop_region.to_dict(),
"projected_point_count": self.projected_point_count,
"crop_name": self.crop_name,
}
@dataclass(frozen=True, slots=True)
class GeometrySemanticRoiFrame:
frame_index: int
rois: tuple[GeometrySemanticRoi, ...]
not_projected_observations: tuple[ObstacleObservation, ...]
def build_geometry_semantic_rois(
*,
frame: GeometryFrame,
observations: tuple[ObstacleObservation, ...],
profile: GeometrySemanticRoiProfile | None = None,
) -> GeometrySemanticRoiFrame:
"""Project geometry-owned point identities into bounded semantic crops."""
selected_profile = profile or GeometrySemanticRoiProfile()
if frame.frame_index < 0 or not frame.surface_valid:
raise GeometrySemanticRoiError("geometry semantic ROI frame is unavailable")
if any(
item.frame_id != f"frame-{frame.frame_index:06d}"
or item.metric_geometry is None
or not item.occupied_support
for item in observations
):
raise GeometrySemanticRoiError("geometry semantic ROI observations are incompatible")
projected = project_map_points_kb4(
frame.points_map,
position_map_xyz=frame.sensor_position_map,
orientation_map_from_lidar_xyzw=frame.sensor_orientation_xyzw,
profile=frame.projection,
)
rois: list[GeometrySemanticRoi] = []
missing: list[ObstacleObservation] = []
for observation in observations:
source_ids = np.asarray(observation.source_point_ids, dtype=np.int64)
selected = np.isin(projected.source_indices, source_ids)
pixels = projected.pixels_xy[selected]
if pixels.shape[0] < selected_profile.minimum_projected_points:
missing.append(observation)
continue
x_min, y_min = pixels.min(axis=0)
x_max, y_max = pixels.max(axis=0)
core = _nonempty_region(
float(x_min),
float(y_min),
float(x_max),
float(y_max),
width=frame.projection.width,
height=frame.projection.height,
)
crop = _crop_region(
core,
width=frame.projection.width,
height=frame.projection.height,
profile=selected_profile,
)
ordinal = len(rois)
roi_id = f"{observation.frame_id}:geometry-roi-{ordinal:03d}"
rois.append(
GeometrySemanticRoi(
roi_id=roi_id,
frame_index=frame.frame_index,
observation=observation,
core_region=core,
crop_region=crop,
projected_point_count=int(pixels.shape[0]),
crop_name=(
f"frame-{frame.frame_index:06d}-geometry-roi-{ordinal:03d}.png"
),
)
)
return GeometrySemanticRoiFrame(
frame_index=frame.frame_index,
rois=tuple(rois),
not_projected_observations=tuple(missing),
)
def materialize_geometry_semantic_crop(
*,
image_path: Path,
roi: GeometrySemanticRoi,
destination: Path,
) -> None:
"""Write one lossless crop while retaining raw-image coordinate lineage."""
try:
with Image.open(image_path.resolve(strict=True)) as opened:
if opened.size != (800, 600):
raise GeometrySemanticRoiError("semantic crop source raster changed")
image = opened.convert("RGB")
crop = image.crop(_integer_box(roi.crop_region))
except OSError as exc:
raise GeometrySemanticRoiError("semantic crop source cannot be read") from exc
target = destination.resolve()
if target.exists() or target.name != roi.crop_name:
raise GeometrySemanticRoiError("semantic crop destination is invalid")
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
crop.save(target, format="PNG", optimize=False)
def select_geometry_roi_detections(
roi: GeometrySemanticRoi,
fusion: SemanticFusionResult,
) -> tuple[OpenVocabularyDetection, ...]:
"""Select the semantic cluster covering the geometry-owned core, if any."""
center_x = (roi.core_region.x_min + roi.core_region.x_max) / 2.0
center_y = (roi.core_region.y_min + roi.core_region.y_max) / 2.0
candidates = []
for binding in fusion.bindings:
region = binding.proposal.region
center_inside = (
region.x_min <= center_x <= region.x_max
and region.y_min <= center_y <= region.y_max
)
core_coverage = _intersection_area(region, roi.core_region) / _area(roi.core_region)
if center_inside or core_coverage >= 0.25:
candidates.append((binding.proposal.objectness, core_coverage, binding))
if not candidates:
return ()
selected = max(candidates, key=lambda item: (item[0], item[1], item[2].proposal.proposal_id))
return selected[2].detections
def _nonempty_region(
x_min: float,
y_min: float,
x_max: float,
y_max: float,
*,
width: int,
height: int,
) -> BoundingRegion2D:
if x_max - x_min < 1.0:
center = (x_min + x_max) / 2.0
x_min, x_max = center - 0.5, center + 0.5
if y_max - y_min < 1.0:
center = (y_min + y_max) / 2.0
y_min, y_max = center - 0.5, center + 0.5
x_min = max(0.0, min(x_min, width - 1.0))
y_min = max(0.0, min(y_min, height - 1.0))
x_max = min(float(width), max(x_max, x_min + 1.0))
y_max = min(float(height), max(y_max, y_min + 1.0))
return BoundingRegion2D(x_min, y_min, x_max, y_max)
def _crop_region(
core: BoundingRegion2D,
*,
width: int,
height: int,
profile: GeometrySemanticRoiProfile,
) -> BoundingRegion2D:
core_width = core.x_max - core.x_min
core_height = core.y_max - core.y_min
padding_x = max(profile.minimum_padding_pixels, core_width * profile.padding_fraction)
padding_y = max(profile.minimum_padding_pixels, core_height * profile.padding_fraction)
target_width = min(
width,
max(profile.minimum_crop_width, math.ceil(core_width + 2.0 * padding_x)),
)
target_height = min(
height,
max(profile.minimum_crop_height, math.ceil(core_height + 2.0 * padding_y)),
)
center_x = (core.x_min + core.x_max) / 2.0
center_y = (core.y_min + core.y_max) / 2.0
x_min = int(round(center_x - target_width / 2.0))
y_min = int(round(center_y - target_height / 2.0))
x_min = min(max(0, x_min), width - target_width)
y_min = min(max(0, y_min), height - target_height)
return BoundingRegion2D(
float(x_min),
float(y_min),
float(x_min + target_width),
float(y_min + target_height),
)
def _integer_box(region: BoundingRegion2D) -> tuple[int, int, int, int]:
return (
int(region.x_min),
int(region.y_min),
int(region.x_max),
int(region.y_max),
)
def _area(region: BoundingRegion2D) -> float:
return (region.x_max - region.x_min) * (region.y_max - region.y_min)
def _intersection_area(left: BoundingRegion2D, right: BoundingRegion2D) -> float:
return max(0.0, min(left.x_max, right.x_max) - max(left.x_min, right.x_min)) * max(
0.0,
min(left.y_max, right.y_max) - max(left.y_min, right.y_min),
)
__all__ = [
"GeometrySemanticRoi",
"GeometrySemanticRoiError",
"GeometrySemanticRoiFrame",
"GeometrySemanticRoiProfile",
"build_geometry_semantic_rois",
"materialize_geometry_semantic_crop",
"select_geometry_roi_detections",
]
src/k1link/perception/geometry_semantic_shadow_replay.py
+580
View File
@@ -0,0 +1,580 @@
"""Immutable geometry-first M48S semantic shadow replay."""
from __future__ import annotations
import hashlib
import json
import shutil
import uuid
from collections import Counter
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Final
from .detector_replay_result import (
read_detector_replay_result,
require_m4_detector_replay_acceptance,
)
from .geometry import Ravnoves00GeometryAssociationProvider, RecordedGeometryStore
from .geometry_semantic_roi import (
GeometrySemanticRoi,
build_geometry_semantic_rois,
select_geometry_roi_detections,
)
from .graph_validation import validate_observations
from .object_understanding import (
ObjectUnderstanding,
SemanticResolution,
load_object_semantic_vocabulary,
)
from .open_vocabulary_semantics import (
OpenVocabularyDetection,
fuse_open_vocabulary_detections,
load_open_vocabulary_semantic_profile,
parse_tao_grounding_dino_labels,
understand_geometry_observation,
)
from .semantic_shadow_replay import semantic_replay_packet
from .yolox_object_detector import load_valid_fov_mask
GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA: Final = (
"missioncore.m48s-geometry-semantic-shadow-replay/v0"
)
GEOMETRY_SEMANTIC_SHADOW_FRAME_SCHEMA: Final = (
"missioncore.m48s-geometry-semantic-shadow-frame/v0"
)
GEOMETRY_SEMANTIC_ROI_PACKAGE_SCHEMA: Final = (
"missioncore.m48s-geometry-semantic-roi-package/v0"
)
RESULT_PREFIX: Final = "m48s-geometry-semantic-shadow-"
class GeometrySemanticShadowReplayError(RuntimeError):
"""Geometry-first semantic evidence is incomplete or incompatible."""
@dataclass(frozen=True, slots=True)
class GeometrySemanticShadowReplayResult:
result_id: str
result_root: Path
completed: bool
accepted: bool
metrics: dict[str, object]
report: dict[str, object]
manifest: dict[str, object]
def build_geometry_semantic_shadow_replay(
*,
repository_root: Path,
profile_path: Path,
vocabulary_path: Path,
detector_result_root: Path,
roi_package_root: Path,
valid_fov_mask_path: Path,
worker_result_roots: Mapping[str, Path],
frame_indices: tuple[int, ...],
worker_execution: Mapping[str, object],
output_root: Path,
) -> GeometrySemanticShadowReplayResult:
"""Name geometry-owned obstacles while leaving occupancy and authority untouched."""
repository = repository_root.resolve(strict=True)
profile = load_open_vocabulary_semantic_profile(profile_path)
vocabulary = load_object_semantic_vocabulary(vocabulary_path)
if vocabulary.vocabulary_id != profile.vocabulary_id:
raise GeometrySemanticShadowReplayError("semantic profile and vocabulary disagree")
frames = _validate_frame_indices(frame_indices)
roi_root = roi_package_root.resolve(strict=True)
roi_manifest = _read_object(roi_root / "manifest.json", "ROI package manifest")
roi_identity = _validate_roi_manifest(roi_manifest, frames=frames)
crop_names = tuple(
_string(_object(roi, "ROI"), "crop_name")
for frame in _array(roi_identity, "frames")
for roi in _array(_object(frame, "ROI frame"), "rois")
)
_validate_worker_results(worker_result_roots, profile=profile, crop_names=crop_names)
valid_fov_mask = load_valid_fov_mask(
valid_fov_mask_path,
expected_sha256=profile.valid_fov_mask_sha256,
)
detector = read_detector_replay_result(detector_result_root)
require_m4_detector_replay_acceptance(detector)
detector_by_sequence = {item.sequence: item for item in detector.frames}
if any(index not in detector_by_sequence for index in frames):
raise GeometrySemanticShadowReplayError("semantic frame escaped detector timeline")
store = RecordedGeometryStore.from_repository(repository)
geometry = Ravnoves00GeometryAssociationProvider(store=store)
output = output_root.expanduser().absolute()
output.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = output / f".geometry-semantic-shadow.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
totals: Counter[str] = Counter()
resolution_counts: Counter[str] = Counter()
class_counts: Counter[str] = Counter()
try:
frames_path = staging / "frames.jsonl"
with frames_path.open("wb") as ledger:
for frame_index in frames:
detector_frame = detector_by_sequence[frame_index]
if detector_frame.outcome != "completed":
raise GeometrySemanticShadowReplayError("accepted detector frame failed")
packet = semantic_replay_packet(detector_frame.envelope)
observations = geometry.associate(packet, ())
validate_observations(packet, (), observations)
geometry_frame = store.frame(packet)
if geometry_frame is None:
raise GeometrySemanticShadowReplayError("selected geometry frame unavailable")
roi_frame = build_geometry_semantic_rois(
frame=geometry_frame,
observations=observations,
)
stored_frame = _roi_manifest_frame(roi_identity, frame_index)
if [item.to_dict() for item in roi_frame.rois] != [
_roi_without_materialization(_object(item, "stored ROI"))
for item in _array(stored_frame, "rois")
]:
raise GeometrySemanticShadowReplayError("stored ROI geometry changed")
understandings_by_observation: dict[str, ObjectUnderstanding] = {}
roi_documents: list[dict[str, object]] = []
for roi in roi_frame.rois:
detections = _roi_detections(
roi,
worker_result_roots=worker_result_roots,
profile=profile,
)
fusion = fuse_open_vocabulary_detections(
detections,
profile=profile,
vocabulary=vocabulary,
valid_fov_mask=valid_fov_mask,
)
selected = select_geometry_roi_detections(roi, fusion)
understanding = understand_geometry_observation(
roi.observation,
detections=selected,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=detector_frame.envelope.timestamps.monotonic_ns,
)
understandings_by_observation[roi.observation.observation_id] = understanding
totals["roi_count"] += 1
totals["raw_detection_count"] += len(detections)
totals["below_confidence_count"] += fusion.below_confidence_count
totals["invalid_area_count"] += fusion.invalid_area_count
totals["outside_valid_fov_count"] += fusion.outside_valid_fov_count
totals["retained_detection_count"] += fusion.retained_detection_count
totals["fused_proposal_count"] += len(fusion.proposals)
totals["selected_cluster_detection_count"] += len(selected)
totals["roi_with_selected_cluster_count"] += bool(selected)
roi_documents.append(
{
"roi": roi.to_dict(),
"detections": [_detection_document(item) for item in detections],
"fused_proposals": [item.to_dict() for item in fusion.proposals],
"selected_detection_ids": [item.detection_id for item in selected],
"understanding": understanding.to_dict(),
}
)
for observation in roi_frame.not_projected_observations:
understandings_by_observation[observation.observation_id] = (
understand_geometry_observation(
observation,
detections=(),
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=(
detector_frame.envelope.timestamps.monotonic_ns
),
)
)
totals["not_projected_observation_count"] += 1
ordered = tuple(
understandings_by_observation[item.observation_id] for item in observations
)
if len(ordered) != len(observations):
raise GeometrySemanticShadowReplayError("geometry obstacle accounting changed")
for understanding in ordered:
resolution_counts[understanding.semantic.resolution.value] += 1
if (
understanding.semantic.resolution is SemanticResolution.SELECTED
and understanding.semantic.selected_class_id is not None
):
class_counts[understanding.semantic.selected_class_id] += 1
totals["geometry_observation_count"] += len(observations)
frame_document = {
"schema_version": GEOMETRY_SEMANTIC_SHADOW_FRAME_SCHEMA,
"frame_index": frame_index,
"frame_id": detector_frame.envelope.frame_id,
"geometry_observation_count": len(observations),
"roi_count": len(roi_frame.rois),
"not_projected_observation_count": len(
roi_frame.not_projected_observations
),
"rois": roi_documents,
"understandings": [item.to_dict() for item in ordered],
"authority": _false_authority(),
}
ledger.write(_canonical_json(frame_document) + b"\n")
metrics: dict[str, object] = {
"frames": {"requested": len(frames), "completed": len(frames)},
"geometry": {
"observation_count": totals["geometry_observation_count"],
"roi_count": totals["roi_count"],
"not_projected_observation_count": totals[
"not_projected_observation_count"
],
},
"semantic_inference": {
"raw_detection_count": totals["raw_detection_count"],
"below_confidence_count": totals["below_confidence_count"],
"invalid_area_count": totals["invalid_area_count"],
"outside_valid_fov_count": totals["outside_valid_fov_count"],
"retained_detection_count": totals["retained_detection_count"],
"fused_proposal_count": totals["fused_proposal_count"],
"roi_with_selected_cluster_count": totals[
"roi_with_selected_cluster_count"
],
"selected_cluster_detection_count": totals[
"selected_cluster_detection_count"
],
},
"semantics": {
"resolution_counts": dict(sorted(resolution_counts.items())),
"selected_class_counts": dict(sorted(class_counts.items())),
},
"authority": _false_authority(),
}
completed = (
totals["geometry_observation_count"]
== totals["roi_count"] + totals["not_projected_observation_count"]
and sum(resolution_counts.values()) == totals["geometry_observation_count"]
)
identity = {
"schema_version": GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA,
"profile_id": profile.profile_id,
"profile_sha256": _sha256(profile_path),
"vocabulary_id": vocabulary.vocabulary_id,
"vocabulary_sha256": _sha256(vocabulary_path),
"detector_result_id": detector.result_id,
"roi_package_id": _string(roi_manifest, "package_id"),
"roi_package_sha256": _sha256(roi_root / "manifest.json"),
"frame_indices": list(frames),
"worker_execution": dict(worker_execution),
"worker_artifacts": _worker_artifacts(worker_result_roots, profile, crop_names),
"producer_sha256": {
name: _sha256(repository / "src/k1link/perception" / name)
for name in (
"geometry_semantic_roi.py",
"geometry_semantic_shadow_replay.py",
"object_understanding.py",
"open_vocabulary_semantics.py",
)
},
"frames_sha256": _sha256(frames_path),
"metrics": metrics,
"completed": completed,
"accepted": False,
"authority": _false_authority(),
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"{RESULT_PREFIX}{identity_sha256}"
report = {
"schema_version": GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA,
"result_id": result_id,
"created_at_utc": datetime.now(UTC).isoformat(timespec="milliseconds").replace(
"+00:00", "Z"
),
"completed": completed,
"accepted": False,
"metrics": metrics,
"decision": {
"geometry_first_binding_completed": completed,
"semantic_quality_accepted": False,
"navigation_or_safety_accepted": False,
"reason_codes": [
"two-reviewer-independent-truth-unavailable",
"open-vocabulary-geometry-roi-remains-experimental-shadow",
"unknown-and-unresolved-objects-remain-route-around-obstacles",
],
},
"authority": _false_authority(),
}
manifest: dict[str, object] = {
"schema_version": GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"artifacts": {
"frames.jsonl": _sha256(frames_path),
"report.json": hashlib.sha256(_canonical_json(report) + b"\n").hexdigest(),
},
}
(staging / "report.json").write_bytes(_canonical_json(report) + b"\n")
(staging / "manifest.json").write_bytes(_canonical_json(manifest) + b"\n")
destination = output / result_id
if destination.exists():
shutil.rmtree(staging)
else:
staging.rename(destination)
except Exception:
shutil.rmtree(staging, ignore_errors=True)
raise
return GeometrySemanticShadowReplayResult(
result_id=result_id,
result_root=destination,
completed=completed,
accepted=False,
metrics=metrics,
report=report,
manifest=manifest,
)
def read_geometry_semantic_shadow_replay(
result_root: Path,
) -> GeometrySemanticShadowReplayResult:
"""Read and revalidate one immutable geometry-first semantic result."""
root = result_root.resolve(strict=True)
manifest = _read_object(root / "manifest.json", "geometry semantic manifest")
report = _read_object(root / "report.json", "geometry semantic report")
identity = _object(manifest.get("identity"), "geometry semantic identity")
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"{RESULT_PREFIX}{identity_sha256}"
artifacts = _object(manifest.get("artifacts"), "geometry semantic artifacts")
if (
manifest.get("schema_version") != GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA
or manifest.get("identity_sha256") != identity_sha256
or manifest.get("result_id") != result_id
or root.name != result_id
or report.get("schema_version") != GEOMETRY_SEMANTIC_SHADOW_REPLAY_SCHEMA
or report.get("result_id") != result_id
or artifacts
!= {
"frames.jsonl": _sha256(root / "frames.jsonl"),
"report.json": _sha256(root / "report.json"),
}
):
raise GeometrySemanticShadowReplayError("geometry semantic result identity changed")
metrics = _object(report.get("metrics"), "geometry semantic metrics")
completed = report.get("completed")
accepted = report.get("accepted")
if not isinstance(completed, bool) or not isinstance(accepted, bool):
raise GeometrySemanticShadowReplayError("geometry semantic result state changed")
if identity.get("metrics") != metrics or identity.get("completed") != completed:
raise GeometrySemanticShadowReplayError("geometry semantic result accounting changed")
return GeometrySemanticShadowReplayResult(
result_id=result_id,
result_root=root,
completed=completed,
accepted=accepted,
metrics=metrics,
report=report,
manifest=manifest,
)
def _roi_detections(
roi: GeometrySemanticRoi,
*,
worker_result_roots: Mapping[str, Path],
profile: object,
) -> tuple[OpenVocabularyDetection, ...]:
from .open_vocabulary_semantics import OpenVocabularySemanticProfile
if not isinstance(profile, OpenVocabularySemanticProfile):
raise GeometrySemanticShadowReplayError("semantic profile is incompatible")
width = int(roi.crop_region.x_max - roi.crop_region.x_min)
height = int(roi.crop_region.y_max - roi.crop_region.y_min)
detections: list[OpenVocabularyDetection] = []
for group in profile.prompt_groups:
labels = (
worker_result_roots[group.prompt_set_id].resolve(strict=True)
/ "labels"
/ roi.crop_name.replace(".png", ".txt")
)
detections.extend(
parse_tao_grounding_dino_labels(
labels,
source_id=profile.source_id,
frame_id=roi.observation.frame_id,
prompt_set_id=group.prompt_set_id,
profile=profile,
image_width=width,
image_height=height,
offset_x=roi.crop_region.x_min,
offset_y=roi.crop_region.y_min,
detection_scope_id=roi.roi_id,
)
)
return tuple(detections)
def _validate_worker_results(
roots: Mapping[str, Path],
*,
profile: object,
crop_names: tuple[str, ...],
) -> None:
from .open_vocabulary_semantics import OpenVocabularySemanticProfile
if not isinstance(profile, OpenVocabularySemanticProfile):
raise GeometrySemanticShadowReplayError("semantic profile is incompatible")
if set(roots) != {item.prompt_set_id for item in profile.prompt_groups}:
raise GeometrySemanticShadowReplayError("Worker prompt roots changed")
expected = {item.replace(".png", ".txt") for item in crop_names}
for root in roots.values():
resolved = root.resolve(strict=True)
actual = {item.name for item in (resolved / "labels").glob("*.txt")}
if actual != expected:
raise GeometrySemanticShadowReplayError("Worker label coverage changed")
status = (resolved / "status.json").read_text("utf-8")
if '"status": "SUCCESS"' not in status:
raise GeometrySemanticShadowReplayError("Worker semantic run did not succeed")
def _worker_artifacts(
roots: Mapping[str, Path],
profile: object,
crop_names: tuple[str, ...],
) -> dict[str, object]:
from .open_vocabulary_semantics import OpenVocabularySemanticProfile
if not isinstance(profile, OpenVocabularySemanticProfile):
raise GeometrySemanticShadowReplayError("semantic profile is incompatible")
artifacts: dict[str, object] = {}
for group in profile.prompt_groups:
root = roots[group.prompt_set_id].resolve(strict=True)
labels = [
{
"name": crop_name.replace(".png", ".txt"),
"sha256": _sha256(root / "labels" / crop_name.replace(".png", ".txt")),
}
for crop_name in crop_names
]
artifacts[group.prompt_set_id] = {
"status_sha256": _sha256(root / "status.json"),
"experiment_sha256": _sha256(root / "experiment.yaml"),
"labels": labels,
}
return artifacts
def _validate_roi_manifest(
manifest: dict[str, object],
*,
frames: tuple[int, ...],
) -> dict[str, object]:
if manifest.get("schema_version") != GEOMETRY_SEMANTIC_ROI_PACKAGE_SCHEMA:
raise GeometrySemanticShadowReplayError("ROI package schema changed")
identity = _object(manifest.get("identity"), "ROI identity")
if identity.get("frame_indices") != list(frames):
raise GeometrySemanticShadowReplayError("ROI frame selection changed")
if identity.get("roi_count") != 58 or identity.get("geometry_observation_count") != 73:
raise GeometrySemanticShadowReplayError("bounded ROI package accounting changed")
digest = hashlib.sha256(_canonical_json(identity)).hexdigest()
if manifest.get("identity_sha256") != digest or manifest.get("package_id") != (
f"m48s-geometry-semantic-rois-{digest}"
):
raise GeometrySemanticShadowReplayError("ROI package identity changed")
return identity
def _roi_manifest_frame(identity: dict[str, object], frame_index: int) -> dict[str, object]:
matches = [
_object(item, "ROI frame")
for item in _array(identity, "frames")
if _object(item, "ROI frame").get("frame_index") == frame_index
]
if len(matches) != 1:
raise GeometrySemanticShadowReplayError("ROI frame manifest is ambiguous")
return matches[0]
def _roi_without_materialization(value: dict[str, object]) -> dict[str, object]:
return {
key: item
for key, item in value.items()
if key not in {"crop_sha256", "crop_width", "crop_height"}
}
def _validate_frame_indices(value: tuple[int, ...]) -> tuple[int, ...]:
if not value or tuple(sorted(set(value))) != value or any(item < 0 for item in value):
raise GeometrySemanticShadowReplayError("semantic frame selection is invalid")
return value
def _detection_document(value: OpenVocabularyDetection) -> dict[str, object]:
return {
"detection_id": value.detection_id,
"source_id": value.source_id,
"frame_id": value.frame_id,
"prompt_set_id": value.prompt_set_id,
"raw_label": value.raw_label,
"confidence": value.confidence,
"region": value.region.to_dict(),
}
def _read_object(path: Path, label: str) -> dict[str, object]:
try:
value = json.loads(path.resolve(strict=True).read_text("utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise GeometrySemanticShadowReplayError(f"{label} cannot be read") from exc
return _object(value, label)
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise GeometrySemanticShadowReplayError(f"{label} must be an object")
return value
def _array(value: dict[str, object], key: str) -> list[object]:
item = value.get(key)
if not isinstance(item, list):
raise GeometrySemanticShadowReplayError(f"{key} must be an array")
return item
def _string(value: dict[str, object], key: str) -> str:
item = value.get(key)
if not isinstance(item, str) or not item:
raise GeometrySemanticShadowReplayError(f"{key} must be a nonempty string")
return item
def _canonical_json(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.resolve(strict=True).open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _false_authority() -> dict[str, bool]:
return {
"ground_truth": False,
"independent_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
__all__ = [
"GeometrySemanticShadowReplayError",
"GeometrySemanticShadowReplayResult",
"build_geometry_semantic_shadow_replay",
"read_geometry_semantic_shadow_replay",
]
src/k1link/perception/mask_grounding_semantics.py
+436
View File
@@ -0,0 +1,436 @@
"""Fail-closed Mask Grounding DINO evidence and geometry binding.
The mask model is allowed to suggest instance masks and class hypotheses. It
cannot create occupied geometry, and contradictory captions for one pixel mask
remain explicitly ambiguous.
"""
from __future__ import annotations
import hashlib
import math
from collections import defaultdict
from dataclasses import dataclass
from enum import StrEnum
from pathlib import Path
from typing import Final
import numpy as np
import numpy.typing as npt
from .contracts import ObstacleObservation
from .geometry_math import ProjectedPointCloud
MASK_GROUNDING_EVIDENCE_SCHEMA: Final = (
"missioncore.m48s-mask-grounding-dino-evidence/v0"
)
BoolArray = npt.NDArray[np.bool_]
class MaskGroundingSemanticError(ValueError):
"""Mask semantic evidence is malformed or cannot be bound safely."""
class MaskBindingResolution(StrEnum):
SELECTED = "selected"
AMBIGUOUS = "ambiguous"
UNRESOLVED = "unresolved"
class MaskLabelResolution(StrEnum):
SELECTED = "selected"
AMBIGUOUS = "ambiguous"
UNRESOLVED = "unresolved"
@dataclass(frozen=True, slots=True)
class MaskGroundingDetection:
detection_id: str
prompt_set_id: str
class_id: int
class_name: str
confidence: float
box_xyxy: tuple[float, float, float, float]
mask_sha256: str
mask: BoolArray
def __post_init__(self) -> None:
if (
not self.detection_id
or not self.prompt_set_id
or not self.class_name
or self.class_id < 0
or not math.isfinite(self.confidence)
or not 0.0 <= self.confidence <= 1.0
or len(self.box_xyxy) != 4
or not np.isfinite(self.box_xyxy).all()
or self.box_xyxy[2] <= self.box_xyxy[0]
or self.box_xyxy[3] <= self.box_xyxy[1]
or self.mask.shape != (600, 800)
or not self.mask.any()
or len(self.mask_sha256) != 64
):
raise MaskGroundingSemanticError("mask grounding detection is invalid")
frozen = np.asarray(self.mask, dtype=np.bool_).copy()
frozen.setflags(write=False)
object.__setattr__(self, "mask", frozen)
@dataclass(frozen=True, slots=True)
class MaskGroundingEvidence:
source_file_sha256: str
source_pixel_sha256: str
detections: tuple[MaskGroundingDetection, ...]
@dataclass(frozen=True, slots=True)
class MaskInstance:
instance_id: str
detections: tuple[MaskGroundingDetection, ...]
@property
def ranked_labels(self) -> tuple[tuple[str, float], ...]:
confidence_by_label: dict[str, float] = {}
for item in self.detections:
confidence_by_label[item.class_name] = max(
item.confidence,
confidence_by_label.get(item.class_name, 0.0),
)
return tuple(
sorted(
confidence_by_label.items(),
key=lambda item: (-item[1], item[0]),
)
)
@dataclass(frozen=True, slots=True)
class MaskObservationSupport:
observation_id: str
projected_point_count: int
support_fraction: float
@dataclass(frozen=True, slots=True)
class MaskGeometryBinding:
instance_id: str
resolution: MaskBindingResolution
selected_observation_id: str | None
supports: tuple[MaskObservationSupport, ...]
reason_code: str
@dataclass(frozen=True, slots=True)
class MaskLabelDecision:
instance_id: str
resolution: MaskLabelResolution
selected_label: str | None
ranked_labels: tuple[tuple[str, float], ...]
reason_code: str
@dataclass(frozen=True, slots=True)
class MaskGeometryBindingProfile:
minimum_mask_iou: float = 0.9
minimum_projected_points: int = 4
minimum_support_fraction: float = 0.5
maximum_secondary_support_fraction: float = 0.25
minimum_label_confidence: float = 0.3
minimum_label_margin: float = 0.1
def __post_init__(self) -> None:
fractions = (
self.minimum_mask_iou,
self.minimum_support_fraction,
self.maximum_secondary_support_fraction,
self.minimum_label_confidence,
self.minimum_label_margin,
)
if (
self.minimum_projected_points < 1
or not all(math.isfinite(item) and 0.0 <= item <= 1.0 for item in fractions)
or self.maximum_secondary_support_fraction
>= self.minimum_support_fraction
):
raise MaskGroundingSemanticError("mask geometry binding profile is invalid")
def load_mask_grounding_evidence(
path: Path,
*,
prompt_set_id: str,
) -> MaskGroundingEvidence:
"""Load one lossless Worker ledger without allowing object arrays."""
if not prompt_set_id:
raise MaskGroundingSemanticError("mask prompt set id is required")
try:
source = path.resolve(strict=True)
with np.load(source, allow_pickle=False) as archive:
expected = {
"schema_version",
"source_file_sha256",
"source_pixel_sha256",
"class_ids",
"class_names",
"scores",
"boxes_xyxy",
"masks",
}
if set(archive.files) != expected:
raise MaskGroundingSemanticError("mask evidence fields changed")
schema = str(archive["schema_version"].item())
source_file_sha256 = str(archive["source_file_sha256"].item())
source_pixel_sha256 = str(archive["source_pixel_sha256"].item())
class_ids = np.asarray(archive["class_ids"], dtype=np.int64)
class_names = np.asarray(archive["class_names"])
scores = np.asarray(archive["scores"], dtype=np.float64)
boxes = np.asarray(archive["boxes_xyxy"], dtype=np.float64)
masks = np.asarray(archive["masks"], dtype=np.uint8)
except (OSError, ValueError) as exc:
raise MaskGroundingSemanticError("mask evidence cannot be read") from exc
if schema != MASK_GROUNDING_EVIDENCE_SCHEMA:
raise MaskGroundingSemanticError("mask evidence schema changed")
count = int(class_ids.size)
if count == 0 and boxes.shape == (0,):
# Early v0 shadow ledgers encoded an empty list before the writer was
# tightened to (0, 4). No values are inferred by this normalization.
boxes = boxes.reshape((0, 4))
if (
len(source_file_sha256) != 64
or len(source_pixel_sha256) != 64
or class_ids.shape != (count,)
or class_names.shape != (count,)
or scores.shape != (count,)
or boxes.shape != (count, 4)
or masks.shape != (count, 600, 800)
or np.any((masks != 0) & (masks != 1))
):
raise MaskGroundingSemanticError("mask evidence tensors are incompatible")
detections = []
for index in range(count):
mask = masks[index].astype(np.bool_)
mask_sha256 = hashlib.sha256(masks[index].tobytes()).hexdigest()
detections.append(
MaskGroundingDetection(
detection_id=f"{path.stem}:{prompt_set_id}:{index:04d}",
prompt_set_id=prompt_set_id,
class_id=int(class_ids[index]),
class_name=str(class_names[index]),
confidence=float(scores[index]),
box_xyxy=tuple(float(item) for item in boxes[index]), # type: ignore[arg-type]
mask_sha256=mask_sha256,
mask=mask,
)
)
return MaskGroundingEvidence(
source_file_sha256=source_file_sha256,
source_pixel_sha256=source_pixel_sha256,
detections=tuple(detections),
)
def cluster_mask_instances(
detections: tuple[MaskGroundingDetection, ...],
*,
profile: MaskGeometryBindingProfile | None = None,
) -> tuple[MaskInstance, ...]:
"""Collapse caption duplicates into deterministic pixel-owned instances."""
selected_profile = profile or MaskGeometryBindingProfile()
ordered = tuple(sorted(detections, key=lambda item: item.detection_id))
parent = list(range(len(ordered)))
def find(index: int) -> int:
while parent[index] != index:
parent[index] = parent[parent[index]]
index = parent[index]
return index
def union(left: int, right: int) -> None:
left_root, right_root = find(left), find(right)
if left_root != right_root:
parent[max(left_root, right_root)] = min(left_root, right_root)
for left in range(len(ordered)):
for right in range(left):
if _mask_iou(ordered[left].mask, ordered[right].mask) >= (
selected_profile.minimum_mask_iou
):
union(left, right)
grouped: dict[int, list[MaskGroundingDetection]] = defaultdict(list)
for index, detection in enumerate(ordered):
grouped[find(index)].append(detection)
instances = []
for ordinal, members in enumerate(grouped.values()):
member_ids = "\n".join(item.detection_id for item in members).encode()
digest = hashlib.sha256(member_ids).hexdigest()[:16]
instances.append(
MaskInstance(
instance_id=f"mask-instance-{ordinal:03d}-{digest}",
detections=tuple(members),
)
)
return tuple(instances)
def bind_mask_instances_to_geometry(
instances: tuple[MaskInstance, ...],
*,
observations: tuple[ObstacleObservation, ...],
projected: ProjectedPointCloud,
profile: MaskGeometryBindingProfile | None = None,
) -> tuple[MaskGeometryBinding, ...]:
"""Bind masks only to exclusively supported occupied-geometry observations."""
selected_profile = profile or MaskGeometryBindingProfile()
preliminary = tuple(
_bind_instance(
instance,
observations=observations,
projected=projected,
profile=selected_profile,
)
for instance in instances
)
claims: dict[str, list[int]] = defaultdict(list)
for index, binding in enumerate(preliminary):
if binding.selected_observation_id is not None:
claims[binding.selected_observation_id].append(index)
collisions = {index for indices in claims.values() if len(indices) > 1 for index in indices}
return tuple(
MaskGeometryBinding(
instance_id=item.instance_id,
resolution=MaskBindingResolution.AMBIGUOUS,
selected_observation_id=None,
supports=item.supports,
reason_code="observation-claimed-by-multiple-mask-instances",
)
if index in collisions
else item
for index, item in enumerate(preliminary)
)
def resolve_mask_instance_label(
instance: MaskInstance,
*,
profile: MaskGeometryBindingProfile | None = None,
) -> MaskLabelDecision:
"""Resolve a name only when confidence and inter-label margin both pass."""
selected_profile = profile or MaskGeometryBindingProfile()
ranked = instance.ranked_labels
if not ranked or ranked[0][1] < selected_profile.minimum_label_confidence:
return MaskLabelDecision(
instance.instance_id,
MaskLabelResolution.UNRESOLVED,
None,
ranked,
"top-label-below-confidence",
)
runner_up = ranked[1][1] if len(ranked) > 1 else 0.0
if ranked[0][1] - runner_up < selected_profile.minimum_label_margin:
return MaskLabelDecision(
instance.instance_id,
MaskLabelResolution.AMBIGUOUS,
None,
ranked,
"top-label-margin-insufficient",
)
return MaskLabelDecision(
instance.instance_id,
MaskLabelResolution.SELECTED,
ranked[0][0],
ranked,
"confidence-and-margin-passed",
)
def _bind_instance(
instance: MaskInstance,
*,
observations: tuple[ObstacleObservation, ...],
projected: ProjectedPointCloud,
profile: MaskGeometryBindingProfile,
) -> MaskGeometryBinding:
supports = []
for observation in observations:
selected = np.isin(
projected.source_indices,
np.asarray(observation.source_point_ids, dtype=np.int64),
)
pixels = projected.pixels_xy[selected]
if pixels.shape[0] < profile.minimum_projected_points:
continue
integer_pixels = np.floor(pixels).astype(np.int64)
x = integer_pixels[:, 0]
y = integer_pixels[:, 1]
support = max(
float(detection.mask[y, x].mean()) for detection in instance.detections
)
supports.append(
MaskObservationSupport(
observation_id=observation.observation_id,
projected_point_count=int(pixels.shape[0]),
support_fraction=support,
)
)
ordered = tuple(
sorted(
supports,
key=lambda item: (-item.support_fraction, item.observation_id),
)
)
if not ordered or ordered[0].support_fraction < profile.minimum_support_fraction:
return MaskGeometryBinding(
instance.instance_id,
MaskBindingResolution.UNRESOLVED,
None,
ordered,
"occupied-geometry-support-insufficient",
)
if (
len(ordered) > 1
and ordered[1].support_fraction > profile.maximum_secondary_support_fraction
):
return MaskGeometryBinding(
instance.instance_id,
MaskBindingResolution.AMBIGUOUS,
None,
ordered,
"mask-covers-multiple-geometry-observations",
)
return MaskGeometryBinding(
instance.instance_id,
MaskBindingResolution.SELECTED,
ordered[0].observation_id,
ordered,
"exclusive-occupied-geometry-support-passed",
)
def _mask_iou(left: BoolArray, right: BoolArray) -> float:
intersection = int(np.logical_and(left, right).sum())
union = int(np.logical_or(left, right).sum())
return intersection / union if union else 0.0
__all__ = [
"MASK_GROUNDING_EVIDENCE_SCHEMA",
"MaskBindingResolution",
"MaskGeometryBinding",
"MaskGeometryBindingProfile",
"MaskGroundingDetection",
"MaskGroundingEvidence",
"MaskGroundingSemanticError",
"MaskInstance",
"MaskLabelDecision",
"MaskLabelResolution",
"MaskObservationSupport",
"bind_mask_instances_to_geometry",
"cluster_mask_instances",
"load_mask_grounding_evidence",
"resolve_mask_instance_label",
]
src/k1link/perception/object_understanding.py
+929
View File
@@ -0,0 +1,929 @@
"""Versioned semantic, state and advisory-risk projection for an obstacle.
The projection composes an immutable :class:`ObstacleObservation` instead of
changing the strict v1 geometry contract. Semantic identity, observed state,
class priors and advisory risk remain separate claims with explicit evidence.
None of them can create occupancy or acquire navigation, safety or actuation
authority.
"""
from __future__ import annotations
import json
import math
import re
from dataclasses import dataclass
from enum import StrEnum
from pathlib import Path
from typing import Final
from .contracts import FalseAuthority, MotionState, ObstacleObservation
OBJECT_UNDERSTANDING_SCHEMA: Final = "missioncore.object-understanding/v1"
OBJECT_SEMANTIC_VOCABULARY_SCHEMA: Final = "missioncore.object-semantic-vocabulary/v0"
MAX_SEMANTIC_HYPOTHESES: Final = 5
_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,159}$")
_NORMALIZED_LABEL = re.compile(r"^[a-z0-9][a-z0-9_]{0,79}$")
class ObjectUnderstandingError(ValueError):
"""An object-understanding document or vocabulary is incompatible."""
class EvidenceKind(StrEnum):
DETECTOR = "detector"
SEMANTIC_MASK = "semantic-mask"
HUMAN_REVIEW = "human-review"
GEOMETRY = "geometry"
TEMPORAL = "temporal"
POLICY = "policy"
class SemanticResolution(StrEnum):
UNRESOLVED = "unresolved"
SELECTED = "selected"
AMBIGUOUS = "ambiguous"
CONFLICT = "conflict"
class AgencyState(StrEnum):
UNKNOWN = "unknown"
INERT = "inert"
ANIMATE = "animate"
SELF_PROPELLED = "self-propelled"
class StateBasis(StrEnum):
UNKNOWN = "unknown"
OBSERVED = "observed"
CLASS_PRIOR = "class-prior"
FUSED = "fused"
class RiskLevel(StrEnum):
UNKNOWN = "unknown"
LOW = "low"
ELEVATED = "elevated"
HIGH = "high"
CRITICAL = "critical"
class RiskBasis(StrEnum):
UNKNOWN = "unknown"
SEMANTIC_PRIOR = "semantic-prior"
OBSERVED_STATE = "observed-state"
GEOMETRY = "geometry"
FUSED = "fused"
class AdvisoryResponse(StrEnum):
MONITOR = "monitor"
REDUCE_SPEED = "reduce-speed"
YIELD = "yield"
STOP = "stop"
ROUTE_AROUND = "route-around"
@dataclass(frozen=True, slots=True)
class EvidenceProvenance:
"""One source-bound evidence item used by semantic, state or risk claims."""
evidence_id: str
kind: EvidenceKind
source_id: str
frame_id: str
provider_id: str
model_id: str | None
model_revision: str | None
preprocess_id: str | None
prompt_set_id: str | None = None
def __post_init__(self) -> None:
if not isinstance(self.kind, EvidenceKind):
raise ObjectUnderstandingError("evidence kind is invalid")
for value, label in (
(self.evidence_id, "evidence id"),
(self.source_id, "evidence source id"),
(self.frame_id, "evidence frame id"),
(self.provider_id, "evidence provider id"),
):
_identifier(value, label)
for optional_value, label in (
(self.model_id, "evidence model id"),
(self.model_revision, "evidence model revision"),
(self.preprocess_id, "evidence preprocess id"),
(self.prompt_set_id, "evidence prompt-set id"),
):
_optional_identifier(optional_value, label)
if self.kind in {EvidenceKind.DETECTOR, EvidenceKind.SEMANTIC_MASK} and (
self.model_id is None or self.model_revision is None or self.preprocess_id is None
):
raise ObjectUnderstandingError(
"model evidence requires model, revision and preprocess identity"
)
def to_dict(self) -> dict[str, object]:
return {
"evidence_id": self.evidence_id,
"kind": self.kind.value,
"source_id": self.source_id,
"frame_id": self.frame_id,
"provider_id": self.provider_id,
"model_id": self.model_id,
"model_revision": self.model_revision,
"preprocess_id": self.preprocess_id,
"prompt_set_id": self.prompt_set_id,
}
@classmethod
def from_dict(cls, value: object) -> EvidenceProvenance:
document = _object(value, "evidence provenance")
_exact_keys(
document,
{
"evidence_id",
"kind",
"source_id",
"frame_id",
"provider_id",
"model_id",
"model_revision",
"preprocess_id",
"prompt_set_id",
},
"evidence provenance",
)
return cls(
evidence_id=_string(document, "evidence_id"),
kind=_enum(EvidenceKind, document.get("kind"), "evidence kind"),
source_id=_string(document, "source_id"),
frame_id=_string(document, "frame_id"),
provider_id=_string(document, "provider_id"),
model_id=_optional_string(document.get("model_id"), "model id"),
model_revision=_optional_string(document.get("model_revision"), "model revision"),
preprocess_id=_optional_string(document.get("preprocess_id"), "preprocess id"),
prompt_set_id=_optional_string(document.get("prompt_set_id"), "prompt-set id"),
)
@dataclass(frozen=True, slots=True)
class SemanticHypothesis:
"""One ranked canonical class hypothesis, never an occupancy identity."""
rank: int
class_id: str
raw_label: str
confidence: float
evidence_ids: tuple[str, ...]
def __post_init__(self) -> None:
_positive_integer(self.rank, "semantic rank")
_identifier(self.class_id, "semantic class id")
_label(self.raw_label, "raw semantic label")
_confidence(self.confidence, "semantic confidence")
_unique_identifiers(self.evidence_ids, "semantic evidence ids")
def to_dict(self) -> dict[str, object]:
return {
"rank": self.rank,
"class_id": self.class_id,
"raw_label": self.raw_label,
"confidence": self.confidence,
"evidence_ids": list(self.evidence_ids),
}
@classmethod
def from_dict(cls, value: object) -> SemanticHypothesis:
document = _object(value, "semantic hypothesis")
_exact_keys(
document,
{"rank", "class_id", "raw_label", "confidence", "evidence_ids"},
"semantic hypothesis",
)
return cls(
rank=_integer(document, "rank"),
class_id=_string(document, "class_id"),
raw_label=_string(document, "raw_label"),
confidence=_number(document, "confidence"),
evidence_ids=_string_tuple(document.get("evidence_ids"), "evidence ids"),
)
@dataclass(frozen=True, slots=True)
class SemanticDecision:
"""Resolution over ranked hypotheses; ambiguity remains first-class."""
resolution: SemanticResolution
selected_class_id: str | None
selected_confidence: float | None
reason_codes: tuple[str, ...]
def __post_init__(self) -> None:
if not isinstance(self.resolution, SemanticResolution):
raise ObjectUnderstandingError("semantic resolution is invalid")
_optional_identifier(self.selected_class_id, "selected semantic class id")
if self.selected_confidence is not None:
_confidence(self.selected_confidence, "selected semantic confidence")
_unique_identifiers(self.reason_codes, "semantic decision reasons")
has_selection = self.selected_class_id is not None and self.selected_confidence is not None
if self.resolution is SemanticResolution.SELECTED:
if not has_selection:
raise ObjectUnderstandingError("selected semantics require class and confidence")
elif self.selected_class_id is not None or self.selected_confidence is not None:
raise ObjectUnderstandingError("non-selected semantics cannot publish a selected class")
def to_dict(self) -> dict[str, object]:
return {
"resolution": self.resolution.value,
"selected_class_id": self.selected_class_id,
"selected_confidence": self.selected_confidence,
"reason_codes": list(self.reason_codes),
}
@classmethod
def from_dict(cls, value: object) -> SemanticDecision:
document = _object(value, "semantic decision")
_exact_keys(
document,
{
"resolution",
"selected_class_id",
"selected_confidence",
"reason_codes",
},
"semantic decision",
)
return cls(
resolution=_enum(
SemanticResolution,
document.get("resolution"),
"semantic resolution",
),
selected_class_id=_optional_string(
document.get("selected_class_id"), "selected class id"
),
selected_confidence=_optional_number(
document.get("selected_confidence"), "selected confidence"
),
reason_codes=_string_tuple(document.get("reason_codes"), "reason codes"),
)
@dataclass(frozen=True, slots=True)
class ObjectStateEstimate:
"""Observed motion and agency prior, with their bases kept explicit."""
motion: MotionState
motion_confidence: float
agency: AgencyState
agency_basis: StateBasis
evidence_ids: tuple[str, ...]
reason_codes: tuple[str, ...]
def __post_init__(self) -> None:
if not isinstance(self.motion, MotionState):
raise ObjectUnderstandingError("motion state is invalid")
if not isinstance(self.agency, AgencyState):
raise ObjectUnderstandingError("agency state is invalid")
if not isinstance(self.agency_basis, StateBasis):
raise ObjectUnderstandingError("agency basis is invalid")
_confidence(self.motion_confidence, "motion confidence")
_unique_identifiers(self.evidence_ids, "state evidence ids", allow_empty=True)
_unique_identifiers(self.reason_codes, "state reason codes")
if self.agency is AgencyState.UNKNOWN:
if self.agency_basis is not StateBasis.UNKNOWN:
raise ObjectUnderstandingError("unknown agency must retain unknown evidence basis")
elif self.agency_basis is StateBasis.UNKNOWN:
raise ObjectUnderstandingError("agency claim requires an explicit basis")
def to_dict(self) -> dict[str, object]:
return {
"motion": self.motion.value,
"motion_confidence": self.motion_confidence,
"agency": self.agency.value,
"agency_basis": self.agency_basis.value,
"evidence_ids": list(self.evidence_ids),
"reason_codes": list(self.reason_codes),
}
@classmethod
def from_dict(cls, value: object) -> ObjectStateEstimate:
document = _object(value, "object state")
_exact_keys(
document,
{
"motion",
"motion_confidence",
"agency",
"agency_basis",
"evidence_ids",
"reason_codes",
},
"object state",
)
return cls(
motion=_enum(MotionState, document.get("motion"), "motion state"),
motion_confidence=_number(document, "motion_confidence"),
agency=_enum(AgencyState, document.get("agency"), "agency state"),
agency_basis=_enum(StateBasis, document.get("agency_basis"), "agency basis"),
evidence_ids=_string_tuple(document.get("evidence_ids"), "evidence ids"),
reason_codes=_string_tuple(document.get("reason_codes"), "reason codes"),
)
@dataclass(frozen=True, slots=True)
class AdvisoryRiskAssessment:
"""Evidence-qualified risk hint that is never a planner command."""
policy_id: str
level: RiskLevel
confidence: float
basis: RiskBasis
responses: tuple[AdvisoryResponse, ...]
evidence_ids: tuple[str, ...]
reason_codes: tuple[str, ...]
def __post_init__(self) -> None:
if not isinstance(self.level, RiskLevel):
raise ObjectUnderstandingError("risk level is invalid")
if not isinstance(self.basis, RiskBasis):
raise ObjectUnderstandingError("risk basis is invalid")
_identifier(self.policy_id, "risk policy id")
_confidence(self.confidence, "risk confidence")
_unique_enum_values(self.responses, "advisory responses", allow_empty=True)
_unique_identifiers(self.evidence_ids, "risk evidence ids", allow_empty=True)
_unique_identifiers(self.reason_codes, "risk reason codes")
if self.level is RiskLevel.UNKNOWN:
if self.basis is not RiskBasis.UNKNOWN or self.confidence != 0.0:
raise ObjectUnderstandingError(
"unknown risk must retain unknown basis and zero confidence"
)
elif self.basis is RiskBasis.UNKNOWN:
raise ObjectUnderstandingError("risk claim requires an explicit basis")
def to_dict(self) -> dict[str, object]:
return {
"policy_id": self.policy_id,
"level": self.level.value,
"confidence": self.confidence,
"basis": self.basis.value,
"responses": [item.value for item in self.responses],
"evidence_ids": list(self.evidence_ids),
"reason_codes": list(self.reason_codes),
}
@classmethod
def from_dict(cls, value: object) -> AdvisoryRiskAssessment:
document = _object(value, "advisory risk")
_exact_keys(
document,
{
"policy_id",
"level",
"confidence",
"basis",
"responses",
"evidence_ids",
"reason_codes",
},
"advisory risk",
)
return cls(
policy_id=_string(document, "policy_id"),
level=_enum(RiskLevel, document.get("level"), "risk level"),
confidence=_number(document, "confidence"),
basis=_enum(RiskBasis, document.get("basis"), "risk basis"),
responses=tuple(
_enum(AdvisoryResponse, item, "advisory response")
for item in _array(document, "responses")
),
evidence_ids=_string_tuple(document.get("evidence_ids"), "evidence ids"),
reason_codes=_string_tuple(document.get("reason_codes"), "reason codes"),
)
@dataclass(frozen=True, slots=True)
class ObjectUnderstanding:
"""Complete machine projection: geometry, semantics, state, risk and lineage."""
understanding_id: str
vocabulary_id: str
generated_monotonic_ns: int
observation: ObstacleObservation
hypotheses: tuple[SemanticHypothesis, ...]
semantic: SemanticDecision
state: ObjectStateEstimate
risk: AdvisoryRiskAssessment
provenance: tuple[EvidenceProvenance, ...]
authority: FalseAuthority = FalseAuthority()
def __post_init__(self) -> None:
_identifier(self.understanding_id, "understanding id")
_identifier(self.vocabulary_id, "semantic vocabulary id")
_nonnegative_integer(self.generated_monotonic_ns, "generation time")
if not isinstance(self.observation, ObstacleObservation):
raise ObjectUnderstandingError("object geometry observation is invalid")
if (
not isinstance(self.hypotheses, tuple)
or any(not isinstance(item, SemanticHypothesis) for item in self.hypotheses)
or len(self.hypotheses) > MAX_SEMANTIC_HYPOTHESES
):
raise ObjectUnderstandingError("semantic hypothesis set is invalid")
if tuple(item.rank for item in self.hypotheses) != tuple(
range(1, len(self.hypotheses) + 1)
):
raise ObjectUnderstandingError("semantic hypotheses must have contiguous ranks")
class_ids = tuple(item.class_id for item in self.hypotheses)
if len(set(class_ids)) != len(class_ids):
raise ObjectUnderstandingError("semantic hypothesis classes must be unique")
if any(
self.hypotheses[index].confidence < self.hypotheses[index + 1].confidence
for index in range(len(self.hypotheses) - 1)
):
raise ObjectUnderstandingError("semantic hypotheses must be ordered by confidence")
if not isinstance(self.semantic, SemanticDecision):
raise ObjectUnderstandingError("semantic decision is invalid")
if not isinstance(self.state, ObjectStateEstimate):
raise ObjectUnderstandingError("object state is invalid")
if not isinstance(self.risk, AdvisoryRiskAssessment):
raise ObjectUnderstandingError("advisory risk is invalid")
if not isinstance(self.authority, FalseAuthority):
raise ObjectUnderstandingError("object understanding authority is invalid")
if not isinstance(self.provenance, tuple) or any(
not isinstance(item, EvidenceProvenance) for item in self.provenance
):
raise ObjectUnderstandingError("evidence provenance is invalid")
evidence_ids = tuple(item.evidence_id for item in self.provenance)
if len(set(evidence_ids)) != len(evidence_ids):
raise ObjectUnderstandingError("evidence provenance ids must be unique")
if any(
item.source_id != self.observation.source_id
or item.frame_id != self.observation.frame_id
for item in self.provenance
):
raise ObjectUnderstandingError("object evidence escaped its geometry source frame")
known_evidence = set(evidence_ids)
claimed_evidence = (
{evidence_id for item in self.hypotheses for evidence_id in item.evidence_ids}
| set(self.state.evidence_ids)
| set(self.risk.evidence_ids)
)
if claimed_evidence - known_evidence:
raise ObjectUnderstandingError("object claim references unknown evidence")
if (
self.semantic.resolution
in {
SemanticResolution.AMBIGUOUS,
SemanticResolution.CONFLICT,
}
and not self.hypotheses
):
raise ObjectUnderstandingError("ambiguous or conflicting semantics require hypotheses")
if self.semantic.resolution is SemanticResolution.CONFLICT and len(self.hypotheses) < 2:
raise ObjectUnderstandingError("semantic conflict requires two hypotheses")
if self.semantic.resolution is SemanticResolution.SELECTED:
selected = next(
(
item
for item in self.hypotheses
if item.class_id == self.semantic.selected_class_id
),
None,
)
if selected is None or selected.confidence != self.semantic.selected_confidence:
raise ObjectUnderstandingError(
"selected semantics must match one ranked hypothesis"
)
@property
def occupancy_identity(self) -> str:
"""Semantic or risk changes never replace the geometry-owned identity."""
return self.observation.occupancy_identity
def to_dict(self) -> dict[str, object]:
return {
"schema_version": OBJECT_UNDERSTANDING_SCHEMA,
"understanding_id": self.understanding_id,
"vocabulary_id": self.vocabulary_id,
"generated_monotonic_ns": self.generated_monotonic_ns,
"observation": self.observation.to_dict(),
"hypotheses": [item.to_dict() for item in self.hypotheses],
"semantic": self.semantic.to_dict(),
"state": self.state.to_dict(),
"risk": self.risk.to_dict(),
"provenance": [item.to_dict() for item in self.provenance],
"authority": self.authority.to_dict(),
}
@classmethod
def from_dict(cls, value: object) -> ObjectUnderstanding:
document = _contract(
value,
OBJECT_UNDERSTANDING_SCHEMA,
{
"understanding_id",
"vocabulary_id",
"generated_monotonic_ns",
"observation",
"hypotheses",
"semantic",
"state",
"risk",
"provenance",
"authority",
},
"object understanding",
)
return cls(
understanding_id=_string(document, "understanding_id"),
vocabulary_id=_string(document, "vocabulary_id"),
generated_monotonic_ns=_integer(document, "generated_monotonic_ns"),
observation=ObstacleObservation.from_dict(document.get("observation")),
hypotheses=tuple(
SemanticHypothesis.from_dict(item) for item in _array(document, "hypotheses")
),
semantic=SemanticDecision.from_dict(document.get("semantic")),
state=ObjectStateEstimate.from_dict(document.get("state")),
risk=AdvisoryRiskAssessment.from_dict(document.get("risk")),
provenance=tuple(
EvidenceProvenance.from_dict(item) for item in _array(document, "provenance")
),
authority=FalseAuthority.from_dict(document.get("authority")),
)
@dataclass(frozen=True, slots=True)
class CanonicalObjectClass:
"""One class in the bounded experimental object vocabulary."""
class_id: str
parent_id: str | None
aliases: tuple[str, ...]
agency_prior: AgencyState
risk_traits: tuple[str, ...]
def __post_init__(self) -> None:
if not isinstance(self.agency_prior, AgencyState):
raise ObjectUnderstandingError("canonical agency prior is invalid")
_identifier(self.class_id, "canonical class id")
_optional_identifier(self.parent_id, "canonical parent id")
if not self.aliases:
raise ObjectUnderstandingError("canonical class aliases must be nonempty")
normalized = tuple(normalize_raw_label(item) for item in self.aliases)
if len(set(normalized)) != len(normalized):
raise ObjectUnderstandingError("canonical class aliases must be unique")
_unique_identifiers(self.risk_traits, "class risk traits", allow_empty=True)
@dataclass(frozen=True, slots=True)
class ObjectSemanticVocabulary:
"""Loaded executable vocabulary profile; not a runtime ontology service."""
vocabulary_id: str
status: str
scope: str
classes: tuple[CanonicalObjectClass, ...]
max_hypotheses: int
def __post_init__(self) -> None:
_identifier(self.vocabulary_id, "vocabulary id")
if self.status != "experimental":
raise ObjectUnderstandingError("object vocabulary must remain experimental")
_identifier(self.scope, "vocabulary scope")
if not 1 <= self.max_hypotheses <= MAX_SEMANTIC_HYPOTHESES:
raise ObjectUnderstandingError("vocabulary top-k bound is invalid")
if not self.classes:
raise ObjectUnderstandingError("object vocabulary must declare classes")
by_id = {item.class_id: item for item in self.classes}
if len(by_id) != len(self.classes):
raise ObjectUnderstandingError("canonical class ids must be unique")
for item in self.classes:
if item.parent_id is not None and item.parent_id not in by_id:
raise ObjectUnderstandingError("canonical class parent is undeclared")
seen = {item.class_id}
parent_id = item.parent_id
while parent_id is not None:
if parent_id in seen:
raise ObjectUnderstandingError("canonical class hierarchy is cyclic")
seen.add(parent_id)
parent_id = by_id[parent_id].parent_id
aliases = [normalize_raw_label(alias) for item in self.classes for alias in item.aliases]
if len(set(aliases)) != len(aliases):
raise ObjectUnderstandingError("canonical aliases must be globally unique")
def class_definition(self, class_id: str) -> CanonicalObjectClass:
for item in self.classes:
if item.class_id == class_id:
return item
raise ObjectUnderstandingError("canonical class is undeclared")
def resolve_label(self, raw_label: str) -> str | None:
normalized = normalize_raw_label(raw_label)
for item in self.classes:
if normalized in {normalize_raw_label(alias) for alias in item.aliases}:
return item.class_id
return None
def ancestors(self, class_id: str) -> tuple[str, ...]:
by_id = {item.class_id: item for item in self.classes}
current = self.class_definition(class_id)
result: list[str] = []
while current.parent_id is not None:
result.append(current.parent_id)
current = by_id[current.parent_id]
return tuple(result)
def load_object_semantic_vocabulary(path: Path) -> ObjectSemanticVocabulary:
"""Load and fail-close an executable vocabulary profile."""
try:
document = json.loads(path.expanduser().resolve(strict=True).read_text("utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ObjectUnderstandingError("object vocabulary cannot be read") from exc
root = _object(document, "object semantic vocabulary")
_exact_keys(
root,
{
"schema_version",
"vocabulary_id",
"status",
"scope",
"classes",
"policies",
},
"object semantic vocabulary",
)
if root.get("schema_version") != OBJECT_SEMANTIC_VOCABULARY_SCHEMA:
raise ObjectUnderstandingError("object vocabulary schema is incompatible")
policies = _object(root.get("policies"), "object vocabulary policies")
_exact_keys(
policies,
{
"occupancy_independent_of_semantics",
"unknown_preserves_obstacle",
"class_prior_is_not_observed_state",
"risk_is_advisory_only",
"planner_command_authority",
"max_hypotheses",
},
"object vocabulary policies",
)
required_true = (
"occupancy_independent_of_semantics",
"unknown_preserves_obstacle",
"class_prior_is_not_observed_state",
"risk_is_advisory_only",
)
if (
any(policies.get(key) is not True for key in required_true)
or policies.get("planner_command_authority") is not False
):
raise ObjectUnderstandingError("object vocabulary authority policy changed")
classes: list[CanonicalObjectClass] = []
for raw in _array(root, "classes"):
row = _object(raw, "canonical object class")
_exact_keys(
row,
{"class_id", "parent_id", "aliases", "agency_prior", "risk_traits"},
"canonical object class",
)
classes.append(
CanonicalObjectClass(
class_id=_string(row, "class_id"),
parent_id=_optional_string(row.get("parent_id"), "parent id"),
aliases=_string_tuple(row.get("aliases"), "aliases"),
agency_prior=_enum(AgencyState, row.get("agency_prior"), "agency prior"),
risk_traits=_string_tuple(row.get("risk_traits"), "risk traits"),
)
)
return ObjectSemanticVocabulary(
vocabulary_id=_string(root, "vocabulary_id"),
status=_string(root, "status"),
scope=_string(root, "scope"),
classes=tuple(classes),
max_hypotheses=_integer(policies, "max_hypotheses"),
)
def validate_object_understanding(
value: ObjectUnderstanding,
vocabulary: ObjectSemanticVocabulary,
) -> None:
"""Validate canonical class references without changing the document."""
if not isinstance(value, ObjectUnderstanding):
raise ObjectUnderstandingError("object understanding is invalid")
if value.vocabulary_id != vocabulary.vocabulary_id:
raise ObjectUnderstandingError("object understanding vocabulary changed")
declared = {item.class_id for item in vocabulary.classes}
referenced = {item.class_id for item in value.hypotheses}
if value.semantic.selected_class_id is not None:
referenced.add(value.semantic.selected_class_id)
if referenced - declared:
raise ObjectUnderstandingError("object understanding uses undeclared classes")
if len(value.hypotheses) > vocabulary.max_hypotheses:
raise ObjectUnderstandingError("object understanding exceeds vocabulary top-k")
def normalize_raw_label(value: str) -> str:
"""Normalize a provider label only for alias lookup, never as class truth."""
_label(value, "raw semantic label")
normalized = re.sub(r"[_\s-]+", "_", value.strip().lower())
if _NORMALIZED_LABEL.fullmatch(normalized) is None:
raise ObjectUnderstandingError("raw semantic label cannot be normalized")
return normalized
def _contract(
value: object,
schema: str,
fields: set[str],
label: str,
) -> dict[str, object]:
document = _object(value, label)
_exact_keys(document, {"schema_version", *fields}, label)
if document.get("schema_version") != schema:
raise ObjectUnderstandingError(f"{label} schema is incompatible")
return document
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise ObjectUnderstandingError(f"{label} must be an object")
return value
def _exact_keys(document: dict[str, object], expected: set[str], label: str) -> None:
if set(document) != expected:
raise ObjectUnderstandingError(f"{label} fields are incompatible")
def _array(document: dict[str, object], key: str) -> list[object]:
value = document.get(key)
if not isinstance(value, list):
raise ObjectUnderstandingError(f"{key} must be an array")
return value
def _string(document: dict[str, object], key: str) -> str:
return _string_value(document.get(key), key)
def _string_value(value: object, label: str) -> str:
if not isinstance(value, str) or not value:
raise ObjectUnderstandingError(f"{label} must be a nonempty string")
return value
def _optional_string(value: object, label: str) -> str | None:
if value is None:
return None
return _string_value(value, label)
def _integer(document: dict[str, object], key: str) -> int:
value = document.get(key)
if not isinstance(value, int) or isinstance(value, bool):
raise ObjectUnderstandingError(f"{key} must be an integer")
return value
def _number(document: dict[str, object], key: str) -> float:
value = document.get(key)
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
):
raise ObjectUnderstandingError(f"{key} must be finite")
return float(value)
def _optional_number(value: object, label: str) -> float | None:
if value is None:
return None
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
):
raise ObjectUnderstandingError(f"{label} must be finite")
return float(value)
def _identifier(value: str, label: str) -> None:
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
raise ObjectUnderstandingError(f"{label} is not a safe identifier")
def _optional_identifier(value: str | None, label: str) -> None:
if value is not None:
_identifier(value, label)
def _label(value: str, label: str) -> None:
if (
not isinstance(value, str)
or not value
or value != value.strip()
or len(value) > 120
or any(ord(character) < 32 for character in value)
):
raise ObjectUnderstandingError(f"{label} is invalid")
def _nonnegative_integer(value: object, label: str) -> int:
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
raise ObjectUnderstandingError(f"{label} must be a nonnegative integer")
return value
def _positive_integer(value: object, label: str) -> int:
result = _nonnegative_integer(value, label)
if result == 0:
raise ObjectUnderstandingError(f"{label} must be positive")
return result
def _confidence(value: object, label: str) -> float:
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
or not 0.0 <= float(value) <= 1.0
):
raise ObjectUnderstandingError(f"{label} must be within [0, 1]")
return float(value)
def _unique_identifiers(
values: tuple[str, ...],
label: str,
*,
allow_empty: bool = False,
) -> None:
if (not values and not allow_empty) or len(set(values)) != len(values):
raise ObjectUnderstandingError(f"{label} must be unique")
for value in values:
_identifier(value, label)
def _unique_enum_values(
values: tuple[AdvisoryResponse, ...],
label: str,
*,
allow_empty: bool,
) -> None:
if (not values and not allow_empty) or len(set(values)) != len(values):
raise ObjectUnderstandingError(f"{label} must be unique")
if any(not isinstance(value, AdvisoryResponse) for value in values):
raise ObjectUnderstandingError(f"{label} are invalid")
def _string_tuple(value: object, label: str) -> tuple[str, ...]:
if not isinstance(value, list):
raise ObjectUnderstandingError(f"{label} must be an array")
return tuple(_string_value(item, label) for item in value)
def _enum[ENUM: StrEnum](
enum_type: type[ENUM],
value: object,
label: str,
) -> ENUM:
if not isinstance(value, str):
raise ObjectUnderstandingError(f"{label} must be a string")
try:
return enum_type(value)
except ValueError as exc:
raise ObjectUnderstandingError(f"{label} is incompatible") from exc
__all__ = [
"MAX_SEMANTIC_HYPOTHESES",
"OBJECT_SEMANTIC_VOCABULARY_SCHEMA",
"OBJECT_UNDERSTANDING_SCHEMA",
"AdvisoryResponse",
"AdvisoryRiskAssessment",
"AgencyState",
"CanonicalObjectClass",
"EvidenceKind",
"EvidenceProvenance",
"ObjectSemanticVocabulary",
"ObjectStateEstimate",
"ObjectUnderstanding",
"ObjectUnderstandingError",
"RiskBasis",
"RiskLevel",
"SemanticDecision",
"SemanticHypothesis",
"SemanticResolution",
"StateBasis",
"load_object_semantic_vocabulary",
"normalize_raw_label",
"validate_object_understanding",
]
src/k1link/perception/open_vocabulary_semantics.py
+784
View File
@@ -0,0 +1,784 @@
"""Bounded open-vocabulary detections projected onto immutable obstacle geometry.
The adapter deliberately consumes detections in the raw KB4 image coordinate
space used by the admitted geometry provider. It fuses prompt collisions into
one spatial proposal, retains ranked canonical hypotheses and never turns a
semantic label into occupancy, risk authority or a planner command.
"""
from __future__ import annotations
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Final
import numpy as np
from numpy.typing import NDArray
from .contracts import BoundingRegion2D, MotionState, ObjectProposal2D, ObstacleObservation
from .object_understanding import (
AdvisoryResponse,
AdvisoryRiskAssessment,
AgencyState,
EvidenceKind,
EvidenceProvenance,
ObjectSemanticVocabulary,
ObjectStateEstimate,
ObjectUnderstanding,
RiskBasis,
RiskLevel,
SemanticDecision,
SemanticHypothesis,
SemanticResolution,
StateBasis,
normalize_raw_label,
validate_object_understanding,
)
OPEN_VOCABULARY_SEMANTIC_SHADOW_PROFILE_SCHEMA: Final = (
"missioncore.open-vocabulary-semantic-shadow-profile/v0"
)
TAO_GROUNDING_DINO_TRAILING_FIELDS: Final = 15
class OpenVocabularySemanticError(ValueError):
"""An open-vocabulary profile, label ledger or binding is incompatible."""
@dataclass(frozen=True, slots=True)
class PromptGroup:
prompt_set_id: str
captions: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class OpenVocabularySemanticProfile:
profile_id: str
session_id: str
source_id: str
camera_source_id: str
coordinate_space: str
width: int
height: int
valid_fov_result_id: str
valid_fov_mask_sha256: str
valid_fov_fill_value: int
provider_id: str
provider_name: str
model_id: str
model_revision: str
model_sha256: str
engine_sha256: str
container_reference: str
container_image_id: str
preprocess_id: str
engine_input_width: int
engine_input_height: int
minimum_input_confidence: float
minimum_box_area_fraction: float
maximum_box_area_fraction: float
minimum_valid_fov_fraction: float
require_center_inside_valid_fov: bool
fusion_iou_threshold: float
selected_minimum_confidence: float
selected_minimum_margin: float
max_hypotheses: int
prompt_groups: tuple[PromptGroup, ...]
vocabulary_id: str
risk_policy_id: str
def __post_init__(self) -> None:
if self.coordinate_space != "raw-kb4" or (self.width, self.height) != (800, 600):
raise OpenVocabularySemanticError("semantic shadow must remain in raw KB4 800x600")
if self.width < 1 or self.height < 1:
raise OpenVocabularySemanticError("semantic shadow raster is invalid")
if self.valid_fov_fill_value != 114 or len(self.valid_fov_mask_sha256) != 64:
raise OpenVocabularySemanticError("semantic valid-FOV preprocessing changed")
if (self.engine_input_width, self.engine_input_height) != (960, 544):
raise OpenVocabularySemanticError("semantic engine input raster changed")
for value, label in (
(self.minimum_input_confidence, "minimum input confidence"),
(self.minimum_box_area_fraction, "minimum box area fraction"),
(self.maximum_box_area_fraction, "maximum box area fraction"),
(self.minimum_valid_fov_fraction, "minimum valid-FOV fraction"),
(self.fusion_iou_threshold, "fusion IoU threshold"),
(self.selected_minimum_confidence, "selection confidence"),
(self.selected_minimum_margin, "selection margin"),
):
if not math.isfinite(value) or not 0.0 <= value <= 1.0:
raise OpenVocabularySemanticError(f"{label} must be within [0, 1]")
if (
self.minimum_box_area_fraction <= 0.0
or self.maximum_box_area_fraction <= self.minimum_box_area_fraction
or not 1 <= self.max_hypotheses <= 5
or not self.prompt_groups
or self.require_center_inside_valid_fov is not True
):
raise OpenVocabularySemanticError("semantic shadow postprocessing is invalid")
prompt_ids = tuple(item.prompt_set_id for item in self.prompt_groups)
if len(set(prompt_ids)) != len(prompt_ids):
raise OpenVocabularySemanticError("semantic prompt-set ids are duplicated")
captions = tuple(
normalize_raw_label(caption)
for group in self.prompt_groups
for caption in group.captions
)
if len(set(captions)) != len(captions):
raise OpenVocabularySemanticError("semantic captions are duplicated")
def prompt_group(self, prompt_set_id: str) -> PromptGroup:
for item in self.prompt_groups:
if item.prompt_set_id == prompt_set_id:
return item
raise OpenVocabularySemanticError("semantic prompt set is undeclared")
@dataclass(frozen=True, slots=True)
class OpenVocabularyDetection:
detection_id: str
source_id: str
frame_id: str
prompt_set_id: str
raw_label: str
confidence: float
region: BoundingRegion2D
def __post_init__(self) -> None:
if not self.detection_id or not self.source_id or not self.frame_id:
raise OpenVocabularySemanticError("semantic detection identity is invalid")
normalize_raw_label(self.raw_label)
if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0:
raise OpenVocabularySemanticError("semantic detection confidence is invalid")
@property
def evidence_id(self) -> str:
return f"evidence:{self.detection_id}"
@dataclass(frozen=True, slots=True)
class SemanticProposalBinding:
proposal: ObjectProposal2D
detections: tuple[OpenVocabularyDetection, ...]
@dataclass(frozen=True, slots=True)
class SemanticFusionResult:
bindings: tuple[SemanticProposalBinding, ...]
input_detection_count: int
below_confidence_count: int
invalid_area_count: int
outside_valid_fov_count: int
@property
def proposals(self) -> tuple[ObjectProposal2D, ...]:
return tuple(item.proposal for item in self.bindings)
@property
def retained_detection_count(self) -> int:
return sum(len(item.detections) for item in self.bindings)
def load_open_vocabulary_semantic_profile(path: Path) -> OpenVocabularySemanticProfile:
"""Load the experimental shadow profile and fail closed on authority drift."""
try:
root = json.loads(path.expanduser().resolve(strict=True).read_text("utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise OpenVocabularySemanticError("semantic shadow profile cannot be read") from exc
if not isinstance(root, dict) or root.get("schema_version") != (
OPEN_VOCABULARY_SEMANTIC_SHADOW_PROFILE_SCHEMA
):
raise OpenVocabularySemanticError("semantic shadow profile schema changed")
expected_root = {
"schema_version",
"profile_id",
"status",
"source",
"provider",
"postprocessing",
"prompt_groups",
"semantic_vocabulary_id",
"risk_policy_id",
"authority",
}
if set(root) != expected_root or root.get("status") != "experimental-shadow":
raise OpenVocabularySemanticError("semantic shadow profile fields changed")
source = _object(root.get("source"), "semantic source")
provider = _object(root.get("provider"), "semantic provider")
postprocessing = _object(root.get("postprocessing"), "semantic postprocessing")
authority = _object(root.get("authority"), "semantic authority")
if authority != _false_authority():
raise OpenVocabularySemanticError("semantic shadow acquired authority")
raw_groups = root.get("prompt_groups")
if not isinstance(raw_groups, list):
raise OpenVocabularySemanticError("semantic prompt groups must be an array")
groups: list[PromptGroup] = []
for raw_group in raw_groups:
group = _object(raw_group, "semantic prompt group")
if set(group) != {"prompt_set_id", "captions"}:
raise OpenVocabularySemanticError("semantic prompt group fields changed")
raw_captions = group.get("captions")
if (
not isinstance(raw_captions, list)
or not raw_captions
or any(not isinstance(item, str) or not item for item in raw_captions)
):
raise OpenVocabularySemanticError("semantic prompt captions are invalid")
groups.append(
PromptGroup(
prompt_set_id=_string(group, "prompt_set_id"),
captions=tuple(raw_captions),
)
)
return OpenVocabularySemanticProfile(
profile_id=_string(root, "profile_id"),
session_id=_string(source, "session_id"),
source_id=_string(source, "source_id"),
camera_source_id=_string(source, "camera_source_id"),
coordinate_space=_string(source, "coordinate_space"),
width=_integer(source, "width"),
height=_integer(source, "height"),
valid_fov_result_id=_string(source, "valid_fov_result_id"),
valid_fov_mask_sha256=_string(source, "valid_fov_mask_sha256"),
valid_fov_fill_value=_integer(source, "valid_fov_fill_value"),
provider_id=_string(provider, "provider_id"),
provider_name=_string(provider, "name"),
model_id=_string(provider, "model_id"),
model_revision=_string(provider, "model_revision"),
model_sha256=_string(provider, "model_sha256"),
engine_sha256=_string(provider, "engine_sha256"),
container_reference=_string(provider, "container_reference"),
container_image_id=_string(provider, "container_image_id"),
preprocess_id=_string(provider, "preprocess_id"),
engine_input_width=_integer(provider, "engine_input_width"),
engine_input_height=_integer(provider, "engine_input_height"),
minimum_input_confidence=_number(postprocessing, "minimum_input_confidence"),
minimum_box_area_fraction=_number(postprocessing, "minimum_box_area_fraction"),
maximum_box_area_fraction=_number(postprocessing, "maximum_box_area_fraction"),
minimum_valid_fov_fraction=_number(postprocessing, "minimum_valid_fov_fraction"),
require_center_inside_valid_fov=_boolean(postprocessing, "require_center_inside_valid_fov"),
fusion_iou_threshold=_number(postprocessing, "fusion_iou_threshold"),
selected_minimum_confidence=_number(postprocessing, "selected_minimum_confidence"),
selected_minimum_margin=_number(postprocessing, "selected_minimum_margin"),
max_hypotheses=_integer(postprocessing, "max_hypotheses"),
prompt_groups=tuple(groups),
vocabulary_id=_string(root, "semantic_vocabulary_id"),
risk_policy_id=_string(root, "risk_policy_id"),
)
def parse_tao_grounding_dino_labels(
path: Path,
*,
source_id: str,
frame_id: str,
prompt_set_id: str,
profile: OpenVocabularySemanticProfile,
image_width: int | None = None,
image_height: int | None = None,
offset_x: float = 0.0,
offset_y: float = 0.0,
detection_scope_id: str | None = None,
) -> tuple[OpenVocabularyDetection, ...]:
"""Parse one TAO label file while preserving original source-image boxes."""
group = profile.prompt_group(prompt_set_id)
allowed = {normalize_raw_label(item) for item in group.captions}
local_width = profile.width if image_width is None else image_width
local_height = profile.height if image_height is None else image_height
if (
local_width < 1
or local_height < 1
or offset_x < 0.0
or offset_y < 0.0
or offset_x + local_width > profile.width
or offset_y + local_height > profile.height
):
raise OpenVocabularySemanticError("semantic label image window is invalid")
scope_id = frame_id if detection_scope_id is None else detection_scope_id
if not scope_id:
raise OpenVocabularySemanticError("semantic detection scope is invalid")
resolved = path.resolve(strict=True)
if resolved.is_symlink() or not resolved.is_file():
raise OpenVocabularySemanticError("semantic label ledger must be a regular file")
detections: list[OpenVocabularyDetection] = []
for line_number, raw_line in enumerate(resolved.read_text("utf-8").splitlines(), start=1):
if not raw_line.strip():
continue
parts = raw_line.split()
if len(parts) <= TAO_GROUNDING_DINO_TRAILING_FIELDS:
raise OpenVocabularySemanticError("TAO semantic label row is incomplete")
label_parts = parts[:-TAO_GROUNDING_DINO_TRAILING_FIELDS]
numeric_parts = parts[-TAO_GROUNDING_DINO_TRAILING_FIELDS:]
raw_label = " ".join(label_parts)
if normalize_raw_label(raw_label) not in allowed:
raise OpenVocabularySemanticError("TAO semantic label escaped its prompt set")
try:
values = tuple(float(item) for item in numeric_parts)
except ValueError as exc:
raise OpenVocabularySemanticError("TAO semantic label row is not numeric") from exc
if any(not math.isfinite(item) for item in values):
raise OpenVocabularySemanticError("TAO semantic label row is not finite")
x_min, y_min, x_max, y_max = values[3:7]
if x_min < 0.0 or y_min < 0.0 or x_max > local_width or y_max > local_height:
raise OpenVocabularySemanticError(
"TAO semantic box is not in the declared raw image coordinate space"
)
detections.append(
OpenVocabularyDetection(
detection_id=(f"{scope_id}:{prompt_set_id}:label-{line_number:04d}"),
source_id=source_id,
frame_id=frame_id,
prompt_set_id=prompt_set_id,
raw_label=raw_label,
confidence=values[-1],
region=BoundingRegion2D(
x_min + offset_x,
y_min + offset_y,
x_max + offset_x,
y_max + offset_y,
),
)
)
return tuple(detections)
def fuse_open_vocabulary_detections(
detections: tuple[OpenVocabularyDetection, ...],
*,
profile: OpenVocabularySemanticProfile,
vocabulary: ObjectSemanticVocabulary,
valid_fov_mask: NDArray[np.bool_],
) -> SemanticFusionResult:
"""Fuse prompt collisions spatially before the geometry provider owns points."""
if vocabulary.vocabulary_id != profile.vocabulary_id:
raise OpenVocabularySemanticError("semantic profile and vocabulary disagree")
if profile.max_hypotheses > vocabulary.max_hypotheses:
raise OpenVocabularySemanticError("semantic profile exceeds vocabulary top-k")
if not detections:
return SemanticFusionResult((), 0, 0, 0, 0)
if valid_fov_mask.shape != (profile.height, profile.width) or valid_fov_mask.dtype != np.bool_:
raise OpenVocabularySemanticError("semantic valid-FOV mask is incompatible")
source_ids = {item.source_id for item in detections}
frame_ids = {item.frame_id for item in detections}
if source_ids != {profile.source_id} or len(frame_ids) != 1:
raise OpenVocabularySemanticError("semantic detections escaped one source frame")
below_confidence = 0
invalid_area = 0
outside_valid_fov = 0
retained: list[OpenVocabularyDetection] = []
raster_area = float(profile.width * profile.height)
for detection in detections:
profile.prompt_group(detection.prompt_set_id)
if vocabulary.resolve_label(detection.raw_label) is None:
raise OpenVocabularySemanticError("semantic label is not in the vocabulary")
if detection.confidence < profile.minimum_input_confidence:
below_confidence += 1
continue
area_fraction = _area(detection.region) / raster_area
if not (
profile.minimum_box_area_fraction <= area_fraction <= profile.maximum_box_area_fraction
):
invalid_area += 1
continue
valid_fraction, center_inside = _valid_fov_support(detection.region, valid_fov_mask)
if valid_fraction < profile.minimum_valid_fov_fraction or (
profile.require_center_inside_valid_fov and not center_inside
):
outside_valid_fov += 1
continue
retained.append(detection)
retained.sort(key=_detection_priority)
clusters: list[list[OpenVocabularyDetection]] = []
for detection in retained:
spatial_candidates = tuple(
(index, max(_iou(detection.region, item.region) for item in cluster))
for index, cluster in enumerate(clusters)
)
match = max(spatial_candidates, key=lambda item: (item[1], -item[0]), default=None)
if match is not None and match[1] >= profile.fusion_iou_threshold:
clusters[match[0]].append(detection)
else:
clusters.append([detection])
frame_id = next(iter(frame_ids))
bindings: list[SemanticProposalBinding] = []
for ordinal, cluster in enumerate(clusters):
ordered = tuple(sorted(cluster, key=_detection_priority))
semantic_candidates = _canonical_candidates(ordered, vocabulary, profile.max_hypotheses)
if not semantic_candidates:
raise OpenVocabularySemanticError("semantic cluster lost all hypotheses")
anchor = ordered[0]
proposal_id = f"semantic-{frame_id}-{ordinal:04d}"
proposal = ObjectProposal2D(
proposal_id=proposal_id,
source_id=anchor.source_id,
frame_id=anchor.frame_id,
region=anchor.region,
objectness=anchor.confidence,
provider_id=profile.provider_id,
model_id=profile.model_id,
preprocess_id=profile.preprocess_id,
semantic_hint=semantic_candidates[0][0],
)
bindings.append(SemanticProposalBinding(proposal=proposal, detections=ordered))
return SemanticFusionResult(
bindings=tuple(bindings),
input_detection_count=len(detections),
below_confidence_count=below_confidence,
invalid_area_count=invalid_area,
outside_valid_fov_count=outside_valid_fov,
)
def bind_object_understandings(
observations: tuple[ObstacleObservation, ...],
*,
bindings: tuple[SemanticProposalBinding, ...],
profile: OpenVocabularySemanticProfile,
vocabulary: ObjectSemanticVocabulary,
generated_monotonic_ns: int,
) -> tuple[ObjectUnderstanding, ...]:
"""Compose semantic hypotheses around geometry-owned obstacle observations."""
by_proposal = {item.proposal.proposal_id: item for item in bindings}
if len(by_proposal) != len(bindings):
raise OpenVocabularySemanticError("semantic proposal bindings are duplicated")
understandings: list[ObjectUnderstanding] = []
seen_proposals: set[str] = set()
for observation in observations:
if not observation.proposal_ids:
understanding = _unknown_understanding(
observation,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=generated_monotonic_ns,
reason="geometry-only-semantic-evidence-unavailable",
)
else:
if len(observation.proposal_ids) != 1:
raise OpenVocabularySemanticError("semantic observation ownership is ambiguous")
proposal_id = observation.proposal_ids[0]
binding = by_proposal.get(proposal_id)
if binding is None:
raise OpenVocabularySemanticError(
"geometry references an unknown semantic proposal"
)
seen_proposals.add(proposal_id)
understanding = _semantic_understanding(
observation,
detections=binding.detections,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=generated_monotonic_ns,
)
validate_object_understanding(understanding, vocabulary)
understandings.append(understanding)
if seen_proposals != set(by_proposal):
raise OpenVocabularySemanticError("a semantic proposal has no geometry observation")
return tuple(understandings)
def understand_geometry_observation(
observation: ObstacleObservation,
*,
detections: tuple[OpenVocabularyDetection, ...],
profile: OpenVocabularySemanticProfile,
vocabulary: ObjectSemanticVocabulary,
generated_monotonic_ns: int,
) -> ObjectUnderstanding:
"""Attach ROI semantics directly to geometry without changing its identity."""
if not detections:
result = _unknown_understanding(
observation,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=generated_monotonic_ns,
reason="geometry-roi-semantic-evidence-unavailable",
)
else:
result = _semantic_understanding(
observation,
detections=detections,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=generated_monotonic_ns,
)
validate_object_understanding(result, vocabulary)
return result
def _semantic_understanding(
observation: ObstacleObservation,
*,
detections: tuple[OpenVocabularyDetection, ...],
profile: OpenVocabularySemanticProfile,
vocabulary: ObjectSemanticVocabulary,
generated_monotonic_ns: int,
) -> ObjectUnderstanding:
candidates = _canonical_candidates(
detections,
vocabulary,
profile.max_hypotheses,
)
hypotheses: list[SemanticHypothesis] = []
referenced: dict[str, OpenVocabularyDetection] = {}
for rank, (class_id, raw_label, confidence, evidence) in enumerate(candidates, start=1):
for detection in evidence:
referenced[detection.evidence_id] = detection
hypotheses.append(
SemanticHypothesis(
rank=rank,
class_id=class_id,
raw_label=raw_label,
confidence=confidence,
evidence_ids=tuple(item.evidence_id for item in evidence),
)
)
top = hypotheses[0]
runner_up = hypotheses[1] if len(hypotheses) > 1 else None
if top.confidence < profile.selected_minimum_confidence:
semantic = SemanticDecision(
resolution=SemanticResolution.UNRESOLVED,
selected_class_id=None,
selected_confidence=None,
reason_codes=("top-hypothesis-below-selection-threshold",),
)
elif (
runner_up is not None
and top.confidence - runner_up.confidence < profile.selected_minimum_margin
):
semantic = SemanticDecision(
resolution=SemanticResolution.AMBIGUOUS,
selected_class_id=None,
selected_confidence=None,
reason_codes=("top-hypothesis-margin-insufficient",),
)
else:
semantic = SemanticDecision(
resolution=SemanticResolution.SELECTED,
selected_class_id=top.class_id,
selected_confidence=top.confidence,
reason_codes=("top-hypothesis-qualified",),
)
if semantic.resolution is SemanticResolution.SELECTED:
definition = vocabulary.class_definition(top.class_id)
agency = definition.agency_prior
if agency is AgencyState.UNKNOWN:
agency_basis = StateBasis.UNKNOWN
state_evidence: tuple[str, ...] = ()
state_reasons: tuple[str, ...] = (
"observed-motion-unavailable",
"selected-class-has-no-agency-prior",
)
else:
agency_basis = StateBasis.CLASS_PRIOR
state_evidence = top.evidence_ids
state_reasons = (
"observed-motion-unavailable",
"agency-from-selected-class-prior",
)
else:
agency = AgencyState.UNKNOWN
agency_basis = StateBasis.UNKNOWN
state_evidence = ()
state_reasons = ("observed-motion-and-resolved-agency-unavailable",)
provenance = tuple(
EvidenceProvenance(
evidence_id=detection.evidence_id,
kind=EvidenceKind.DETECTOR,
source_id=observation.source_id,
frame_id=observation.frame_id,
provider_id=profile.provider_id,
model_id=profile.model_id,
model_revision=profile.model_revision,
preprocess_id=profile.preprocess_id,
prompt_set_id=detection.prompt_set_id,
)
for detection in sorted(referenced.values(), key=lambda item: item.evidence_id)
)
return ObjectUnderstanding(
understanding_id=f"{observation.observation_id}:understanding",
vocabulary_id=vocabulary.vocabulary_id,
generated_monotonic_ns=generated_monotonic_ns,
observation=observation,
hypotheses=tuple(hypotheses),
semantic=semantic,
state=ObjectStateEstimate(
motion=MotionState.UNKNOWN,
motion_confidence=0.0,
agency=agency,
agency_basis=agency_basis,
evidence_ids=state_evidence,
reason_codes=state_reasons,
),
risk=_unknown_risk(profile, "semantic-shadow-has-no-risk-authority"),
provenance=provenance,
)
def _unknown_understanding(
observation: ObstacleObservation,
*,
profile: OpenVocabularySemanticProfile,
vocabulary: ObjectSemanticVocabulary,
generated_monotonic_ns: int,
reason: str,
) -> ObjectUnderstanding:
return ObjectUnderstanding(
understanding_id=f"{observation.observation_id}:understanding",
vocabulary_id=vocabulary.vocabulary_id,
generated_monotonic_ns=generated_monotonic_ns,
observation=observation,
hypotheses=(),
semantic=SemanticDecision(
resolution=SemanticResolution.UNRESOLVED,
selected_class_id=None,
selected_confidence=None,
reason_codes=(reason,),
),
state=ObjectStateEstimate(
motion=MotionState.UNKNOWN,
motion_confidence=0.0,
agency=AgencyState.UNKNOWN,
agency_basis=StateBasis.UNKNOWN,
evidence_ids=(),
reason_codes=("state-evidence-unavailable",),
),
risk=_unknown_risk(profile, "unknown-object-remains-route-around-obstacle"),
provenance=(),
)
def _unknown_risk(
profile: OpenVocabularySemanticProfile,
reason: str,
) -> AdvisoryRiskAssessment:
return AdvisoryRiskAssessment(
policy_id=profile.risk_policy_id,
level=RiskLevel.UNKNOWN,
confidence=0.0,
basis=RiskBasis.UNKNOWN,
responses=(AdvisoryResponse.ROUTE_AROUND,),
evidence_ids=(),
reason_codes=(reason,),
)
def _canonical_candidates(
detections: tuple[OpenVocabularyDetection, ...],
vocabulary: ObjectSemanticVocabulary,
maximum: int,
) -> tuple[tuple[str, str, float, tuple[OpenVocabularyDetection, ...]], ...]:
by_class: dict[str, list[OpenVocabularyDetection]] = {}
for detection in detections:
class_id = vocabulary.resolve_label(detection.raw_label)
if class_id is None:
raise OpenVocabularySemanticError("semantic label is not in the vocabulary")
by_class.setdefault(class_id, []).append(detection)
candidates: list[tuple[str, str, float, tuple[OpenVocabularyDetection, ...]]] = []
for class_id, evidence in by_class.items():
ordered = tuple(sorted(evidence, key=_detection_priority))
candidates.append((class_id, ordered[0].raw_label, ordered[0].confidence, ordered))
candidates.sort(key=lambda item: (-item[2], item[0], normalize_raw_label(item[1])))
return tuple(candidates[:maximum])
def _detection_priority(item: OpenVocabularyDetection) -> tuple[float, float, str]:
return (-item.confidence, _area(item.region), item.detection_id)
def _area(region: BoundingRegion2D) -> float:
return (region.x_max - region.x_min) * (region.y_max - region.y_min)
def _iou(left: BoundingRegion2D, right: BoundingRegion2D) -> float:
x_min = max(left.x_min, right.x_min)
y_min = max(left.y_min, right.y_min)
x_max = min(left.x_max, right.x_max)
y_max = min(left.y_max, right.y_max)
intersection = max(0.0, x_max - x_min) * max(0.0, y_max - y_min)
union = _area(left) + _area(right) - intersection
return 0.0 if union <= 0.0 else intersection / union
def _valid_fov_support(
region: BoundingRegion2D,
mask: NDArray[np.bool_],
) -> tuple[float, bool]:
height, width = mask.shape
x_min = int(np.clip(math.floor(region.x_min), 0, width))
y_min = int(np.clip(math.floor(region.y_min), 0, height))
x_max = int(np.clip(math.ceil(region.x_max), 0, width))
y_max = int(np.clip(math.ceil(region.y_max), 0, height))
area = max(0, x_max - x_min) * max(0, y_max - y_min)
if area == 0:
return 0.0, False
valid_fraction = float(np.count_nonzero(mask[y_min:y_max, x_min:x_max])) / area
center_x = int(np.clip(round((region.x_min + region.x_max) / 2.0), 0, width - 1))
center_y = int(np.clip(round((region.y_min + region.y_max) / 2.0), 0, height - 1))
return valid_fraction, bool(mask[center_y, center_x])
def _false_authority() -> dict[str, bool]:
return {
"ground_truth": False,
"independent_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise OpenVocabularySemanticError(f"{label} must be an object")
return value
def _string(document: dict[str, object], key: str) -> str:
value = document.get(key)
if not isinstance(value, str) or not value:
raise OpenVocabularySemanticError(f"{key} must be a nonempty string")
return value
def _integer(document: dict[str, object], key: str) -> int:
value = document.get(key)
if not isinstance(value, int) or isinstance(value, bool):
raise OpenVocabularySemanticError(f"{key} must be an integer")
return value
def _number(document: dict[str, object], key: str) -> float:
value = document.get(key)
if not isinstance(value, (int, float)) or isinstance(value, bool):
raise OpenVocabularySemanticError(f"{key} must be numeric")
return float(value)
def _boolean(document: dict[str, object], key: str) -> bool:
value = document.get(key)
if not isinstance(value, bool):
raise OpenVocabularySemanticError(f"{key} must be boolean")
return value
__all__ = [
"OPEN_VOCABULARY_SEMANTIC_SHADOW_PROFILE_SCHEMA",
"OpenVocabularyDetection",
"OpenVocabularySemanticError",
"OpenVocabularySemanticProfile",
"PromptGroup",
"SemanticFusionResult",
"SemanticProposalBinding",
"bind_object_understandings",
"fuse_open_vocabulary_detections",
"load_open_vocabulary_semantic_profile",
"parse_tao_grounding_dino_labels",
"understand_geometry_observation",
]
src/k1link/perception/semantic_shadow_replay.py
+768
View File
@@ -0,0 +1,768 @@
"""Immutable bounded M48S replay for raw-KB4 semantic object understanding."""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import uuid
from collections import Counter
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Final
import numpy as np
import yaml
from numpy.typing import NDArray
from PIL import Image
from .baseline import BASELINE_RECORDED_JOB_ID
from .contracts import EvidenceBasis, ObstacleObservation, SourceEnvelope
from .detector_replay_result import (
read_detector_replay_result,
require_m4_detector_replay_acceptance,
)
from .geometry import (
DEFAULT_GEOMETRY_PROFILE_PATH,
Ravnoves00GeometryAssociationProvider,
RecordedGeometryStore,
load_geometry_profile,
)
from .graph_validation import validate_observations
from .object_understanding import (
ObjectUnderstanding,
SemanticResolution,
load_object_semantic_vocabulary,
)
from .open_vocabulary_semantics import (
OpenVocabularyDetection,
OpenVocabularySemanticError,
OpenVocabularySemanticProfile,
bind_object_understandings,
fuse_open_vocabulary_detections,
load_open_vocabulary_semantic_profile,
parse_tao_grounding_dino_labels,
)
from .providers import SourcePacket
from .recorded_source import RECORDED_SOURCE_PACK_ID, RecordedFrameReference
from .yolox_object_detector import load_valid_fov_mask
SEMANTIC_SHADOW_REPLAY_SCHEMA: Final = "missioncore.m48s-semantic-shadow-replay/v0"
SEMANTIC_SHADOW_REPORT_SCHEMA: Final = "missioncore.m48s-semantic-shadow-report/v0"
SEMANTIC_WORKER_EXECUTION_SCHEMA: Final = "missioncore.m48s-semantic-worker-execution/v0"
SEMANTIC_SHADOW_RESULT_PREFIX: Final = "m48s-semantic-shadow-"
SEMANTIC_SHADOW_FRAMES_NAME: Final = "frames.jsonl"
SEMANTIC_SHADOW_REPORT_NAME: Final = "report.json"
SEMANTIC_SHADOW_MANIFEST_NAME: Final = "manifest.json"
class SemanticShadowReplayError(RuntimeError):
"""Bounded semantic shadow evidence is incomplete or internally inconsistent."""
@dataclass(frozen=True, slots=True)
class SemanticShadowReplayResult:
result_id: str
result_root: Path
completed: bool
accepted: bool
metrics: dict[str, object]
report: dict[str, object]
manifest: dict[str, object]
def build_semantic_shadow_replay(
*,
repository_root: Path,
profile_path: Path,
vocabulary_path: Path,
detector_result_root: Path,
source_frames_root: Path,
inference_frames_root: Path,
valid_fov_mask_path: Path,
worker_result_roots: Mapping[str, Path],
worker_identity_path: Path,
frame_indices: tuple[int, ...],
output_root: Path,
) -> SemanticShadowReplayResult:
"""Bind exact Worker labels to admitted geometry for a bounded frame set."""
repository = repository_root.resolve(strict=True)
profile = load_open_vocabulary_semantic_profile(profile_path)
vocabulary = load_object_semantic_vocabulary(vocabulary_path)
if profile.vocabulary_id != vocabulary.vocabulary_id:
raise SemanticShadowReplayError("semantic profile and vocabulary disagree")
frames = _frame_selection(frame_indices)
worker_identity = _validate_worker_identity(
worker_identity_path,
profile=profile,
frame_indices=frames,
)
worker_artifacts = _validate_worker_results(
worker_result_roots,
profile=profile,
frame_indices=frames,
)
valid_fov_mask = load_valid_fov_mask(
valid_fov_mask_path,
expected_sha256=profile.valid_fov_mask_sha256,
)
source_artifacts = _validate_source_frames(
source_frames_root,
profile=profile,
frame_indices=frames,
)
inference_artifacts = _validate_inference_frames(
inference_frames_root,
source_frames_root=source_frames_root,
valid_fov_mask=valid_fov_mask,
profile=profile,
frame_indices=frames,
)
detector = read_detector_replay_result(detector_result_root)
require_m4_detector_replay_acceptance(detector)
detector_by_sequence = {item.sequence: item for item in detector.frames}
if any(index not in detector_by_sequence for index in frames):
raise SemanticShadowReplayError("semantic frame escaped the detector timeline")
geometry_profile = load_geometry_profile(repository / DEFAULT_GEOMETRY_PROFILE_PATH)
store = RecordedGeometryStore.from_repository(repository, profile=geometry_profile)
geometry = Ravnoves00GeometryAssociationProvider(store=store)
output = output_root.expanduser().absolute()
output.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = output / f".semantic-shadow.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700, exist_ok=False)
class_counts: Counter[str] = Counter()
resolution_counts: Counter[str] = Counter()
totals: Counter[str] = Counter()
try:
frames_path = staging / SEMANTIC_SHADOW_FRAMES_NAME
with frames_path.open("wb") as ledger:
for frame_index in frames:
detector_frame = detector_by_sequence[frame_index]
if detector_frame.outcome != "completed":
raise SemanticShadowReplayError("accepted detector source frame failed")
envelope = detector_frame.envelope
detections = _frame_detections(
worker_result_roots,
frame_index=frame_index,
envelope=envelope,
profile=profile,
)
fusion = fuse_open_vocabulary_detections(
detections,
profile=profile,
vocabulary=vocabulary,
valid_fov_mask=valid_fov_mask,
)
packet = semantic_replay_packet(envelope)
observations = geometry.associate(packet, fusion.proposals)
validate_observations(packet, fusion.proposals, observations)
understandings = bind_object_understandings(
observations,
bindings=fusion.bindings,
profile=profile,
vocabulary=vocabulary,
generated_monotonic_ns=envelope.timestamps.monotonic_ns,
)
frame_metrics = _frame_metrics(
detections=detections,
fusion_input_count=fusion.input_detection_count,
below_confidence_count=fusion.below_confidence_count,
invalid_area_count=fusion.invalid_area_count,
outside_valid_fov_count=fusion.outside_valid_fov_count,
retained_detection_count=fusion.retained_detection_count,
proposal_count=len(fusion.proposals),
observations=observations,
understandings=understandings,
)
for key, value in frame_metrics.items():
totals[key] += value
for understanding in understandings:
resolution_counts[understanding.semantic.resolution.value] += 1
if (
understanding.semantic.resolution is SemanticResolution.SELECTED
and understanding.semantic.selected_class_id is not None
):
class_counts[understanding.semantic.selected_class_id] += 1
document = {
"schema_version": "missioncore.m48s-semantic-shadow-frame/v0",
"sequence": frame_index,
"frame_id": envelope.frame_id,
"source_envelope": envelope.to_dict(),
"source_frame": source_artifacts[frame_index],
"inference_frame": inference_artifacts[frame_index],
"detections": [_detection_document(item) for item in detections],
"proposals": [item.to_dict() for item in fusion.proposals],
"understandings": [item.to_dict() for item in understandings],
"metrics": frame_metrics,
"authority": _false_authority(),
}
ledger.write(_canonical_json(document) + b"\n")
metrics = {
"frames": {
"requested": len(frames),
"completed": len(frames),
"source_available": sum(
detector_by_sequence[index].envelope.registered_point_increment.available
for index in frames
),
},
"detections": {
"raw": totals["raw_detection_count"],
"below_confidence": totals["below_confidence_count"],
"invalid_area": totals["invalid_area_count"],
"outside_valid_fov": totals["outside_valid_fov_count"],
"retained": totals["retained_detection_count"],
"fused_proposals": totals["proposal_count"],
},
"geometry": {
"proposal_observations": totals["proposal_observation_count"],
"ranged_proposal_observations": totals["ranged_proposal_observation_count"],
"camera_only_proposal_observations": totals[
"camera_only_proposal_observation_count"
],
"conflict_proposal_observations": totals["conflict_proposal_observation_count"],
"geometry_only_observations": totals["geometry_only_observation_count"],
},
"semantics": {
"resolution_counts": dict(sorted(resolution_counts.items())),
"selected_class_counts": dict(sorted(class_counts.items())),
},
"authority": _false_authority(),
}
completed = (
metrics["frames"]["completed"] == len(frames) # type: ignore[index]
and totals["proposal_observation_count"] == totals["proposal_count"]
and sum(resolution_counts.values())
== totals["proposal_observation_count"] + totals["geometry_only_observation_count"]
)
frames_sha256 = _file_sha256(frames_path)
detector_identity = _object(detector.manifest.get("identity"), "detector identity")
identity = {
"schema_version": SEMANTIC_SHADOW_REPLAY_SCHEMA,
"profile_id": profile.profile_id,
"profile_sha256": _file_sha256(profile_path),
"vocabulary_id": vocabulary.vocabulary_id,
"vocabulary_sha256": _file_sha256(vocabulary_path),
"frame_indices": list(frames),
"detector_result_id": detector.result_id,
"detector_frames_sha256": detector_identity.get("frames_sha256"),
"geometry_profile_id": geometry_profile.profile_id,
"geometry_profile_sha256": geometry_profile.profile_sha256,
"source_pack_id": geometry_profile.source_pack_id,
"source_pack_sha256": geometry_profile.source_pack_sha256,
"valid_fov": {
"result_id": profile.valid_fov_result_id,
"mask_sha256": profile.valid_fov_mask_sha256,
"fill_value": profile.valid_fov_fill_value,
},
"source_frames": [source_artifacts[index] for index in frames],
"inference_frames": [inference_artifacts[index] for index in frames],
"worker_execution": worker_identity,
"worker_execution_sha256": _file_sha256(worker_identity_path),
"worker_artifacts": worker_artifacts,
"producer_sha256": {
name: _file_sha256(repository / "src/k1link/perception" / name)
for name in (
"object_understanding.py",
"open_vocabulary_semantics.py",
"semantic_shadow_replay.py",
)
},
"frames_sha256": frames_sha256,
"metrics": metrics,
"completed": completed,
"accepted": False,
"authority": _false_authority(),
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"{SEMANTIC_SHADOW_RESULT_PREFIX}{identity_sha256}"
created = datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
report = {
"schema_version": SEMANTIC_SHADOW_REPORT_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"created_at_utc": created,
"status": (
"completed-shadow-evidence-not-accepted"
if completed
else "rejected-incomplete-shadow"
),
"completed": completed,
"accepted": False,
"metrics": metrics,
"decision": {
"raw_kb4_coordinate_binding_completed": completed,
"semantic_quality_accepted": False,
"risk_policy_accepted": False,
"navigation_or_safety_accepted": False,
"next_gate": "M48S human-reviewed semantic object quality slice",
},
"known_limits": [
"bounded 11-frame diagnostic slice is not representative route truth",
"Grounding DINO confidence is not calibrated class probability",
"motion is intentionally unresolved without temporal evidence",
"risk remains unknown and route-around advisory only",
"TAO performed network metadata requests during tokenizer startup",
],
"authority": _false_authority(),
}
report_path = staging / SEMANTIC_SHADOW_REPORT_NAME
_write_json(report_path, report)
manifest = {
"schema_version": SEMANTIC_SHADOW_REPLAY_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": created,
"completed": completed,
"accepted": False,
"artifacts": [
_artifact(frames_path, "semantic-shadow-frames"),
_artifact(report_path, "semantic-shadow-report"),
],
"authority": _false_authority(),
}
_write_json(staging / SEMANTIC_SHADOW_MANIFEST_NAME, manifest)
destination = output / result_id
if destination.exists():
shutil.rmtree(staging)
return read_semantic_shadow_replay(destination)
os.replace(staging, destination)
return read_semantic_shadow_replay(destination)
except (OpenVocabularySemanticError, OSError, ValueError) as exc:
shutil.rmtree(staging, ignore_errors=True)
raise SemanticShadowReplayError("semantic shadow replay failed closed") from exc
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
def read_semantic_shadow_replay(root: Path) -> SemanticShadowReplayResult:
"""Read and verify one immutable semantic shadow replay result."""
resolved = root.resolve(strict=True)
if resolved.is_symlink() or not resolved.name.startswith(SEMANTIC_SHADOW_RESULT_PREFIX):
raise SemanticShadowReplayError("semantic shadow result root is invalid")
manifest = _read_json(resolved / SEMANTIC_SHADOW_MANIFEST_NAME)
identity = _object(manifest.get("identity"), "semantic shadow identity")
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
if (
manifest.get("schema_version") != SEMANTIC_SHADOW_REPLAY_SCHEMA
or manifest.get("result_id") != resolved.name
or manifest.get("identity_sha256") != identity_sha256
or resolved.name != f"{SEMANTIC_SHADOW_RESULT_PREFIX}{identity_sha256}"
or manifest.get("accepted") is not False
or manifest.get("authority") != _false_authority()
):
raise SemanticShadowReplayError("semantic shadow manifest changed")
frames_path = resolved / SEMANTIC_SHADOW_FRAMES_NAME
report_path = resolved / SEMANTIC_SHADOW_REPORT_NAME
if _file_sha256(frames_path) != identity.get("frames_sha256") or not _artifact_matches(
manifest.get("artifacts"), frames_path, report_path
):
raise SemanticShadowReplayError("semantic shadow artifacts changed")
report = _read_json(report_path)
if (
report.get("schema_version") != SEMANTIC_SHADOW_REPORT_SCHEMA
or report.get("result_id") != resolved.name
or report.get("identity_sha256") != identity_sha256
or report.get("metrics") != identity.get("metrics")
or report.get("completed") != identity.get("completed")
or report.get("accepted") is not False
or report.get("authority") != _false_authority()
):
raise SemanticShadowReplayError("semantic shadow report changed")
metrics = _object(identity.get("metrics"), "semantic shadow metrics")
return SemanticShadowReplayResult(
result_id=resolved.name,
result_root=resolved,
completed=identity.get("completed") is True,
accepted=False,
metrics=metrics,
report=report,
manifest=manifest,
)
def _frame_selection(value: tuple[int, ...]) -> tuple[int, ...]:
if (
not value
or tuple(sorted(set(value))) != value
or any(isinstance(item, bool) or item < 0 or item >= 4489 for item in value)
):
raise SemanticShadowReplayError("semantic shadow frame selection is invalid")
return value
def _validate_worker_identity(
path: Path,
*,
profile: OpenVocabularySemanticProfile,
frame_indices: tuple[int, ...],
) -> dict[str, object]:
document = _read_json(path.resolve(strict=True))
if (
document.get("schema_version") != SEMANTIC_WORKER_EXECUTION_SCHEMA
or document.get("model_sha256") != profile.model_sha256
or document.get("engine_sha256") != profile.engine_sha256
or document.get("container_reference") != profile.container_reference
or document.get("container_image_id") != profile.container_image_id
or document.get("coordinate_space") != "raw-kb4-800x600"
or document.get("preprocess_id") != profile.preprocess_id
or document.get("valid_fov_mask_sha256") != profile.valid_fov_mask_sha256
or document.get("frame_indices") != list(frame_indices)
or document.get("authority") != _false_authority()
):
raise SemanticShadowReplayError("semantic Worker execution identity changed")
prompt_runs = _object(document.get("prompt_runs"), "semantic prompt runs")
if set(prompt_runs) != {item.prompt_set_id for item in profile.prompt_groups} or any(
_object(value, "semantic prompt run").get("status") != "SUCCESS"
for value in prompt_runs.values()
):
raise SemanticShadowReplayError("semantic Worker prompt run failed")
return document
def _validate_worker_results(
roots: Mapping[str, Path],
*,
profile: OpenVocabularySemanticProfile,
frame_indices: tuple[int, ...],
) -> list[dict[str, object]]:
if set(roots) != {item.prompt_set_id for item in profile.prompt_groups}:
raise SemanticShadowReplayError("semantic Worker result groups changed")
expected_names = {f"frame-{index:06d}.txt" for index in frame_indices}
result: list[dict[str, object]] = []
for group in profile.prompt_groups:
root = roots[group.prompt_set_id].resolve(strict=True)
if root.is_symlink() or not root.is_dir():
raise SemanticShadowReplayError("semantic Worker result root is invalid")
status_path = root / "status.json"
status_rows = _read_jsonl(status_path)
if not status_rows or status_rows[-1].get("status") != "SUCCESS":
raise SemanticShadowReplayError("semantic Worker result did not succeed")
experiment_path = root / "experiment.yaml"
try:
experiment = yaml.safe_load(experiment_path.read_text("utf-8"))
except (OSError, yaml.YAMLError) as exc:
raise SemanticShadowReplayError("semantic Worker experiment cannot be read") from exc
experiment_root = _object(experiment, "semantic Worker experiment")
dataset = _object(experiment_root.get("dataset"), "semantic Worker dataset")
sources = _object(dataset.get("infer_data_sources"), "semantic Worker sources")
inference = _object(experiment_root.get("inference"), "semantic Worker inference")
if (
sources.get("captions") != list(group.captions)
or sources.get("image_dir") != ["/workspace/probe/input"]
or inference.get("conf_threshold") != profile.minimum_input_confidence
or inference.get("input_width") != profile.engine_input_width
or inference.get("input_height") != profile.engine_input_height
or not str(inference.get("trt_engine", "")).endswith(
"/grounding_dino_swin_tiny_commercial_fp16.engine"
)
):
raise SemanticShadowReplayError("semantic Worker experiment drifted")
labels_root = root / "labels"
label_paths = tuple(sorted(labels_root.glob("*.txt")))
if {item.name for item in label_paths} != expected_names or any(
item.is_symlink() or not item.is_file() for item in label_paths
):
raise SemanticShadowReplayError("semantic Worker label accounting changed")
result.append(
{
"prompt_set_id": group.prompt_set_id,
"experiment_sha256": _file_sha256(experiment_path),
"status_sha256": _file_sha256(status_path),
"labels": [
{
"name": item.name,
"byte_length": item.stat().st_size,
"sha256": _file_sha256(item),
}
for item in label_paths
],
}
)
return result
def _validate_source_frames(
root: Path,
*,
profile: OpenVocabularySemanticProfile,
frame_indices: tuple[int, ...],
) -> dict[int, dict[str, object]]:
resolved = root.resolve(strict=True)
expected_names = {f"frame-{index:06d}.jpg" for index in frame_indices}
paths = tuple(sorted(resolved.glob("*.jpg")))
if {item.name for item in paths} != expected_names:
raise SemanticShadowReplayError("semantic raw frame accounting changed")
result: dict[int, dict[str, object]] = {}
for index in frame_indices:
path = resolved / f"frame-{index:06d}.jpg"
if path.is_symlink() or not path.is_file():
raise SemanticShadowReplayError("semantic raw frame is invalid")
try:
with Image.open(path) as image:
image.verify()
with Image.open(path) as image:
dimensions = image.size
image_format = image.format
except OSError as exc:
raise SemanticShadowReplayError("semantic raw frame cannot be decoded") from exc
if dimensions != (profile.width, profile.height) or image_format != "JPEG":
raise SemanticShadowReplayError("semantic raw frame coordinate space changed")
result[index] = {
"source_frame_index": index,
"camera_sequence": index + 1,
"name": path.name,
"width": profile.width,
"height": profile.height,
"byte_length": path.stat().st_size,
"sha256": _file_sha256(path),
"coordinate_space": profile.coordinate_space,
"exact_source_frame": True,
}
return result
def _validate_inference_frames(
root: Path,
*,
source_frames_root: Path,
valid_fov_mask: NDArray[np.bool_],
profile: OpenVocabularySemanticProfile,
frame_indices: tuple[int, ...],
) -> dict[int, dict[str, object]]:
resolved = root.resolve(strict=True)
source_root = source_frames_root.resolve(strict=True)
expected_names = {f"frame-{index:06d}.png" for index in frame_indices}
paths = tuple(sorted(resolved.glob("*.png")))
if {item.name for item in paths} != expected_names:
raise SemanticShadowReplayError("semantic inference frame accounting changed")
result: dict[int, dict[str, object]] = {}
for index in frame_indices:
path = resolved / f"frame-{index:06d}.png"
source_path = source_root / f"frame-{index:06d}.jpg"
if path.is_symlink() or not path.is_file():
raise SemanticShadowReplayError("semantic inference frame is invalid")
try:
with Image.open(path) as opened:
image_format = opened.format
inference = np.asarray(opened.convert("RGB"), dtype=np.uint8)
with Image.open(source_path) as opened:
source = np.asarray(opened.convert("RGB"), dtype=np.uint8)
except OSError as exc:
raise SemanticShadowReplayError("semantic inference frame cannot be decoded") from exc
expected = np.where(
valid_fov_mask[..., None],
source,
profile.valid_fov_fill_value,
).astype(np.uint8)
if (
image_format != "PNG"
or inference.shape != (profile.height, profile.width, 3)
or not np.array_equal(inference, expected)
):
raise SemanticShadowReplayError(
"semantic inference frame valid-FOV preprocessing changed"
)
result[index] = {
"source_frame_index": index,
"name": path.name,
"width": profile.width,
"height": profile.height,
"byte_length": path.stat().st_size,
"sha256": _file_sha256(path),
"coordinate_space": profile.coordinate_space,
"preprocess_id": profile.preprocess_id,
"valid_fov_mask_sha256": profile.valid_fov_mask_sha256,
"valid_fov_fill_value": profile.valid_fov_fill_value,
"derived_from_source_sha256": _file_sha256(source_path),
}
return result
def _frame_detections(
roots: Mapping[str, Path],
*,
frame_index: int,
envelope: SourceEnvelope,
profile: OpenVocabularySemanticProfile,
) -> tuple[OpenVocabularyDetection, ...]:
detections: list[OpenVocabularyDetection] = []
for group in profile.prompt_groups:
labels = (
roots[group.prompt_set_id].resolve(strict=True)
/ "labels"
/ f"frame-{frame_index:06d}.txt"
)
detections.extend(
parse_tao_grounding_dino_labels(
labels,
source_id=envelope.source_id,
frame_id=envelope.frame_id,
prompt_set_id=group.prompt_set_id,
profile=profile,
)
)
return tuple(detections)
def semantic_replay_packet(envelope: SourceEnvelope) -> SourcePacket:
"""Rebuild the exact digest-bound source packet used by semantic replays."""
image = RecordedFrameReference(BASELINE_RECORDED_JOB_ID, envelope.sequence)
geometry = (
RecordedFrameReference(RECORDED_SOURCE_PACK_ID, envelope.sequence)
if envelope.registered_point_increment.available
else None
)
return SourcePacket(
envelope=envelope,
image_payload=image,
registered_point_increment_payload=geometry,
pose_payload=geometry,
)
def _frame_metrics(
*,
detections: tuple[OpenVocabularyDetection, ...],
fusion_input_count: int,
below_confidence_count: int,
invalid_area_count: int,
outside_valid_fov_count: int,
retained_detection_count: int,
proposal_count: int,
observations: tuple[ObstacleObservation, ...],
understandings: tuple[ObjectUnderstanding, ...],
) -> dict[str, int]:
proposal_observations = tuple(item for item in observations if item.proposal_ids)
geometry_only = tuple(item for item in observations if not item.proposal_ids)
if len(detections) != fusion_input_count or len(understandings) != len(observations):
raise SemanticShadowReplayError("semantic frame accounting changed")
ranged = sum(item.metric_geometry is not None for item in proposal_observations)
conflict = sum(item.basis is EvidenceBasis.CONFLICT for item in proposal_observations)
return {
"raw_detection_count": len(detections),
"below_confidence_count": below_confidence_count,
"invalid_area_count": invalid_area_count,
"outside_valid_fov_count": outside_valid_fov_count,
"retained_detection_count": retained_detection_count,
"proposal_count": proposal_count,
"proposal_observation_count": len(proposal_observations),
"ranged_proposal_observation_count": ranged,
"camera_only_proposal_observation_count": (len(proposal_observations) - ranged - conflict),
"conflict_proposal_observation_count": conflict,
"geometry_only_observation_count": len(geometry_only),
}
def _detection_document(value: OpenVocabularyDetection) -> dict[str, object]:
return {
"detection_id": value.detection_id,
"source_id": value.source_id,
"frame_id": value.frame_id,
"prompt_set_id": value.prompt_set_id,
"raw_label": value.raw_label,
"confidence": value.confidence,
"region": value.region.to_dict(),
}
def _artifact(path: Path, role: str) -> dict[str, object]:
return {
"role": role,
"path": path.name,
"byte_length": path.stat().st_size,
"sha256": _file_sha256(path),
}
def _artifact_matches(value: object, frames_path: Path, report_path: Path) -> bool:
if not isinstance(value, list) or len(value) != 2:
return False
expected = {
path.name: (path.stat().st_size, _file_sha256(path)) for path in (frames_path, report_path)
}
seen: set[str] = set()
for item in value:
if not isinstance(item, dict):
return False
path = item.get("path")
if not isinstance(path, str) or path not in expected:
return False
seen.add(path)
if (item.get("byte_length"), item.get("sha256")) != expected[path]:
return False
return seen == set(expected)
def _read_json(path: Path) -> dict[str, object]:
try:
value = json.loads(path.read_text("utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SemanticShadowReplayError(f"cannot read semantic JSON: {path.name}") from exc
return _object(value, path.name)
def _read_jsonl(path: Path) -> list[dict[str, object]]:
try:
rows = [json.loads(line) for line in path.read_text("utf-8").splitlines() if line]
except (OSError, json.JSONDecodeError) as exc:
raise SemanticShadowReplayError(f"cannot read semantic JSONL: {path.name}") from exc
return [_object(item, path.name) for item in rows]
def _write_json(path: Path, value: object) -> None:
path.write_bytes(_canonical_json(value) + b"\n")
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
allow_nan=False,
).encode("utf-8")
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.expanduser().resolve(strict=True).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise SemanticShadowReplayError(f"{label} must be an object")
return value
def _false_authority() -> dict[str, bool]:
return {
"ground_truth": False,
"independent_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
}
__all__ = [
"SEMANTIC_SHADOW_REPLAY_SCHEMA",
"SemanticShadowReplayError",
"SemanticShadowReplayResult",
"build_semantic_shadow_replay",
"read_semantic_shadow_replay",
"semantic_replay_packet",
]