"""Source-bound point materialization for an immutable E30 review pack. The E30 selection pack records which E29 observations require review. This module replays the exact bound E29 method over the exact E10/L2.6 inputs and publishes point-index ownership plus bounded projection context for each selected item. It does not create reviewer decisions or a LAB acceptance. """ from __future__ import annotations import hashlib import json import math import os import re import shutil from dataclasses import asdict, dataclass from datetime import UTC, datetime from pathlib import Path from typing import Any, Final import numpy as np import numpy.typing as npt from k1link.device_plugins.xgrids_k1.analyze.calibrated_projection import ( Kb4ProjectionProfile, ProjectedPointCloud, project_map_points_kb4, ) from .e30_camera_evidence import ( E30CameraEvidenceSource, materialize_e30_camera_frames, open_e30_camera_evidence_source, ) from .lidar_field_review import E10LidarFieldSource from .lidar_local_surface import K1LocalSurfaceV1 from .semantic_geometry_fusion import ( CAMERA_GEOMETRY_FUSION_SCHEMA, CameraGeometryFusionProfile, _claimed_indices, _fusion_frame, _geometry_cluster_supports, _projection_profile, _semantic_support, ) E30_MATERIALIZATION_SCHEMA: Final = "missioncore.e30-evidence-materialization/v2" E30_MATERIALIZATION_ITEM_SCHEMA: Final = ( "missioncore.e30-evidence-materialization-item/v2" ) E30_MATERIALIZATION_INDEX_NAME: Final = "materialized-items.jsonl" E30_MATERIALIZATION_MANIFEST_NAME: Final = "manifest.json" _REVIEW_PACK_ID = re.compile(r"^e30-review-pack-[a-f0-9]{64}$") _REVIEW_ITEM_ID = re.compile(r"^e30-review-item-[a-f0-9]{64}$") _E29_RESULT_ID = re.compile(r"^e29-camera-geometry-[a-f0-9]{64}$") _SOURCE_RESULT_ID = re.compile(r"^e10-integrated-perception-[a-f0-9]{64}$") _SHA256 = re.compile(r"^[a-f0-9]{64}$") FloatArray = npt.NDArray[np.float64] IntArray = npt.NDArray[np.int64] class E30MaterializationError(ValueError): """A source binding or materialized review artifact violates the contract.""" @dataclass(frozen=True, slots=True) class E30MaterializationProfile: profile_id: str = "e30-source-bound-point-materialization/v1" maximum_context_projection_points: int = 6000 def __post_init__(self) -> None: if ( not self.profile_id or len(self.profile_id) > 160 or not 256 <= self.maximum_context_projection_points <= 50_000 ): raise E30MaterializationError("E30 materialization profile is invalid") def to_dict(self) -> dict[str, object]: return { "schema_version": "missioncore.e30-materialization-profile/v1", **asdict(self), "selected_and_candidate_points": "lossless-frame-local-indices", "context_selection": "deterministic-even-index", "camera_evidence": "exact-bound-frame-when-camera-job-is-supplied", "primary_review_view": "camera-with-lidar-projection", "secondary_review_view": "map-frame-3d", "free_space_valid": False, "navigation_or_safety_accepted": False, } DEFAULT_E30_MATERIALIZATION_PROFILE: Final = E30MaterializationProfile() @dataclass(frozen=True, slots=True) class E30Materialization: result_root: Path result_id: str manifest: dict[str, Any] @dataclass(frozen=True, slots=True) class _ReviewPack: root: Path manifest: dict[str, Any] items: tuple[dict[str, Any], ...] items_sha256: str @dataclass(frozen=True, slots=True) class _ItemArrays: selected_source_indices: IntArray selected_points_map_xyz_m: npt.NDArray[np.float32] candidate_source_indices: IntArray candidate_points_map_xyz_m: npt.NDArray[np.float32] projected_source_indices: IntArray projected_points_map_xyz_m: npt.NDArray[np.float32] projected_pixels_xy: npt.NDArray[np.float32] projected_depth_m: npt.NDArray[np.float32] projected_point_class: npt.NDArray[np.uint8] projected_point_height_m: npt.NDArray[np.float32] projected_candidate_mask: npt.NDArray[np.uint8] projected_selected_mask: npt.NDArray[np.uint8] sensor_position_map_xyz_m: npt.NDArray[np.float64] sensor_orientation_map_from_lidar_xyzw: npt.NDArray[np.float64] def to_dict(self) -> dict[str, npt.NDArray[Any]]: return { "selected_source_indices": self.selected_source_indices, "selected_points_map_xyz_m": self.selected_points_map_xyz_m, "candidate_source_indices": self.candidate_source_indices, "candidate_points_map_xyz_m": self.candidate_points_map_xyz_m, "projected_source_indices": self.projected_source_indices, "projected_points_map_xyz_m": self.projected_points_map_xyz_m, "projected_pixels_xy": self.projected_pixels_xy, "projected_depth_m": self.projected_depth_m, "projected_point_class": self.projected_point_class, "projected_point_height_m": self.projected_point_height_m, "projected_candidate_mask": self.projected_candidate_mask, "projected_selected_mask": self.projected_selected_mask, "sensor_position_map_xyz_m": self.sensor_position_map_xyz_m, "sensor_orientation_map_from_lidar_xyzw": ( self.sensor_orientation_map_from_lidar_xyzw ), } def build_e30_materialization( *, review_pack_root: Path, e29_root: Path, source_result_root: Path, source_pack_root: Path, local_surface_root: Path, output_root: Path, camera_job_root: Path | None = None, ffmpeg_path: Path | None = None, profile: E30MaterializationProfile = DEFAULT_E30_MATERIALIZATION_PROFILE, ) -> E30Materialization: """Materialize every selected E30 item from its exact immutable sources.""" review = _read_review_pack(review_pack_root) review_identity = _required_object(review.manifest, "identity") review_source = _required_object(review_identity, "source") e29_result_id = _required_string(review_source, "e29_result_id") if _E29_RESULT_ID.fullmatch(e29_result_id) is None: raise E30MaterializationError("E30 review source E29 id is invalid") e29_result = _child_directory(e29_root, e29_result_id) e29_manifest_path = _regular_file(e29_result, "manifest.json") e29_manifest = _read_json(e29_manifest_path, "E29 manifest") e29_identity = _required_object(e29_manifest, "identity") e29_identity_sha256 = _required_string(e29_manifest, "identity_sha256") if ( e29_manifest.get("schema_version") != CAMERA_GEOMETRY_FUSION_SCHEMA or e29_manifest.get("result_id") != e29_result_id or hashlib.sha256(_canonical_json(e29_identity)).hexdigest() != e29_identity_sha256 or e29_result_id != f"e29-camera-geometry-{e29_identity_sha256}" or review_source.get("e29_identity_sha256") != e29_identity_sha256 ): raise E30MaterializationError("E29 materialization source identity is invalid") _reject_authority(_required_object(e29_identity, "authority"), "E29 identity") source_result_id = _required_string(e29_identity, "source_result_id") source_pack_id = _required_string(e29_identity, "source_pack_id") local_surface_model_id = _required_string( e29_identity, "local_surface_model_id", ) if ( _SOURCE_RESULT_ID.fullmatch(source_result_id) is None or review_source.get("camera_result_id") != source_result_id or review_source.get("lidar_pack_id") != source_pack_id or review_source.get("local_surface_model_id") != local_surface_model_id ): raise E30MaterializationError("E30 linked source identities disagree") source_result = _child_directory(source_result_root, source_result_id) source_result_document_path = _regular_file(source_result, "result.json") source_result_document = _read_json( source_result_document_path, "source perception result", ) source_result_identity = _required_object(source_result_document, "identity") source_result_identity_sha256 = _required_string( source_result_document, "identity_sha256", ) if ( source_result_document.get("result_id") != source_result_id or hashlib.sha256(_canonical_json(source_result_identity)).hexdigest() != source_result_identity_sha256 or source_result_id != f"e10-integrated-perception-{source_result_identity_sha256}" ): raise E30MaterializationError("source perception result identity is invalid") fusion_frames_path = _regular_file(source_result, "fusion-frames.jsonl") fusion_frames_sha256 = _sha256_file(fusion_frames_path) if ( fusion_frames_sha256 != _required_string(e29_identity, "source_fusion_frames_sha256") ): raise E30MaterializationError("source fusion frame digest changed") source = E10LidarFieldSource(_child_directory(source_pack_root, source_pack_id)) surface = K1LocalSurfaceV1( _child_directory(local_surface_root, local_surface_model_id) ) try: _validate_bound_sources(source, surface, e29_identity) fusion_profile = _fusion_profile(e29_identity) projection_profile = _projection_profile(source) if (camera_job_root is None) != (ffmpeg_path is None): raise E30MaterializationError( "camera job and ffmpeg must be supplied together" ) camera_source: E30CameraEvidenceSource | None = None if camera_job_root is not None and ffmpeg_path is not None: camera_source = open_e30_camera_evidence_source( camera_job_root=camera_job_root, ffmpeg_path=ffmpeg_path, expected_session_id=_required_string(source.identity, "session_id"), expected_source_id=projection_profile.source_id, ) selected_frames = _review_items_by_frame(review.items) fusion_frames = _read_selected_fusion_frames( fusion_frames_path, selected_frame_indices=set(selected_frames), source=source, ) identity: dict[str, object] = { "schema_version": E30_MATERIALIZATION_SCHEMA, "review_pack": { "result_id": _required_string(review.manifest, "result_id"), "identity_sha256": _required_string( review.manifest, "identity_sha256", ), "items_sha256": review.items_sha256, "item_count": len(review.items), }, "source": { "e29_result_id": e29_result_id, "e29_manifest_sha256": _sha256_file(e29_manifest_path), "source_result_id": source_result_id, "source_result_sha256": _sha256_file(source_result_document_path), "fusion_frames_sha256": fusion_frames_sha256, "lidar_pack_id": source.pack_id, "lidar_pack_sha256": _source_artifact_sha256(source.manifest), "local_surface_model_id": surface.model_id, "local_surface_sha256": _surface_artifact_sha256(surface.manifest), "source_session_id": _required_string(source.identity, "session_id"), }, "camera_evidence": ( camera_source.identity() if camera_source is not None else None ), "fusion_profile": fusion_profile.to_dict(), "materialization_profile": profile.to_dict(), "projection": { "source_id": projection_profile.source_id, "calibration_slot": projection_profile.calibration_slot, "width": projection_profile.width, "height": projection_profile.height, }, "producer_sha256": _sha256_file(Path(__file__).resolve(strict=True)), "human_review_complete": False, "authority": { "commands_enabled": False, "navigation_or_safety_accepted": False, }, } identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest() result_id = f"e30-materialization-{identity_sha256}" destination = output_root.expanduser().absolute() destination.mkdir(mode=0o700, parents=True, exist_ok=True) result_root = destination / result_id if result_root.exists(): manifest = _read_existing_materialization(result_root, identity) return E30Materialization(result_root, result_id, manifest) staging = destination / f".{result_id}.{os.getpid()}.incomplete" item_root = staging / "items" item_root.mkdir(mode=0o700, parents=True, exist_ok=False) index_path = staging / E30_MATERIALIZATION_INDEX_NAME try: camera_artifacts: dict[int, dict[str, object]] = {} if camera_source is not None: camera_artifacts = materialize_e30_camera_frames( source=camera_source, source_frame_indices=tuple( _item_source_frame_index(item) for item in review.items ), destination_root=staging / "frames", width=projection_profile.width, height=projection_profile.height, ) with index_path.open("x", encoding="utf-8") as index_stream: for item in review.items: frame_index = _item_frame_index(item) arrays, metadata = _materialize_item( item=item, fusion_frame=fusion_frames[frame_index], source=source, surface=surface, fusion_profile=fusion_profile, projection_profile=projection_profile, maximum_context_points=( profile.maximum_context_projection_points ), ) item_id = _required_string(item, "item_id") item_path = item_root / f"{item_id}.npz" array_document = arrays.to_dict() np.savez_compressed( item_path, selected_source_indices=arrays.selected_source_indices, selected_points_map_xyz_m=arrays.selected_points_map_xyz_m, candidate_source_indices=arrays.candidate_source_indices, candidate_points_map_xyz_m=arrays.candidate_points_map_xyz_m, projected_source_indices=arrays.projected_source_indices, projected_points_map_xyz_m=arrays.projected_points_map_xyz_m, projected_pixels_xy=arrays.projected_pixels_xy, projected_depth_m=arrays.projected_depth_m, projected_point_class=arrays.projected_point_class, projected_point_height_m=arrays.projected_point_height_m, projected_candidate_mask=arrays.projected_candidate_mask, projected_selected_mask=arrays.projected_selected_mask, sensor_position_map_xyz_m=arrays.sensor_position_map_xyz_m, sensor_orientation_map_from_lidar_xyzw=( arrays.sensor_orientation_map_from_lidar_xyzw ), ) artifact = { "path": f"items/{item_path.name}", "media_type": "application/x-npz", "byte_length": item_path.stat().st_size, "sha256": _sha256_file(item_path), "logical_sha256": _logical_arrays_sha256(array_document), } index_document = { "schema_version": E30_MATERIALIZATION_ITEM_SCHEMA, "item_id": item_id, "sequence": _required_int(item, "sequence"), "review_key": _required_string(item, "review_key"), "stratum": _required_string(item, "stratum"), "range_bucket": _required_string(item, "range_bucket"), "evidence_binding": _required_object( item, "evidence_binding", ), "e29_locator": _required_object(item, "e29_locator"), "e29_snapshot": _required_object(item, "e29_snapshot"), "materialization": metadata, "artifact": artifact, "camera_frame": camera_artifacts.get( _item_source_frame_index(item) ), "engineering_triage": _engineering_triage( item=item, metadata=metadata, camera_available=( _item_source_frame_index(item) in camera_artifacts ), ), "review": { "state": "unreviewed", "reason_code": None, "notes": None, }, "authority": { "commands_enabled": False, "navigation_or_safety_accepted": False, }, } index_stream.write( _canonical_json(index_document).decode("utf-8") + "\n" ) index_artifact = _artifact( "materialized-items", index_path, "application/x-ndjson", ) manifest = { "schema_version": E30_MATERIALIZATION_SCHEMA, "result_id": result_id, "identity_sha256": identity_sha256, "identity": identity, "created_at_utc": datetime.now(UTC).isoformat(), "classification": "private-derived-perception-review-materialization", "item_count": len(review.items), "camera_evidence_available": camera_source is not None, "human_review_complete": False, "lab_published": False, "artifacts": [index_artifact], "authority": { "commands_enabled": False, "navigation_or_safety_accepted": False, }, } _write_json(staging / E30_MATERIALIZATION_MANIFEST_NAME, manifest) os.replace(staging, result_root) except BaseException: shutil.rmtree(staging, ignore_errors=True) raise return E30Materialization(result_root, result_id, manifest) finally: surface.close() source.close() def _materialize_item( *, item: dict[str, Any], fusion_frame: dict[str, Any], source: E10LidarFieldSource, surface: K1LocalSurfaceV1, fusion_profile: CameraGeometryFusionProfile, projection_profile: Kb4ProjectionProfile, maximum_context_points: int, ) -> tuple[_ItemArrays, dict[str, object]]: frame_index = _item_frame_index(item) start = int(source.arrays["cloud_offsets"][frame_index]) end = int(source.arrays["cloud_offsets"][frame_index + 1]) points_map = np.asarray( source.arrays["cloud_points_map"][start:end], dtype=np.float64, ) point_class = np.asarray( surface.arrays["point_class"][start:end], dtype=np.uint8, ) point_height = np.asarray( surface.arrays["point_height_m"][start:end], dtype=np.float32, ) position = np.asarray( source.arrays["pose_positions_map"][frame_index], dtype=np.float64, ) orientation = np.asarray( source.arrays["pose_quaternions_map_from_lidar"][frame_index], dtype=np.float64, ) source_available = bool(source.arrays["sample_available"][frame_index]) surface_valid = bool(surface.arrays["frame_valid"][frame_index]) projected: ProjectedPointCloud | None = None if source_available and surface_valid: projected = project_map_points_kb4( points_map, position_map_xyz=( float(position[0]), float(position[1]), float(position[2]), ), orientation_map_from_lidar_xyzw=( float(orientation[0]), float(orientation[1]), float(orientation[2]), float(orientation[3]), ), profile=projection_profile, ) objects = _required_array(fusion_frame, "objects") semantic_supports = [ _semantic_support( _object(raw, "fusion object"), projected=projected, frame_points_map=points_map, point_class=point_class, point_height_m=point_height, source_available=source_available, surface_valid=surface_valid, profile=fusion_profile, ) for raw in objects ] geometry_supports = _geometry_cluster_supports( points_map=points_map, point_class=point_class, point_height_m=point_height, sensor_position_map=position, claimed_source_indices=_claimed_indices(semantic_supports), profile=fusion_profile, ) locator = _required_object(item, "e29_locator") snapshot = _required_object(item, "e29_snapshot") locator_kind = _required_string(locator, "kind") if locator_kind == "semantic-observation": observation_index = _required_int(locator, "observation_index") if not 0 <= observation_index < len(semantic_supports): raise E30MaterializationError("semantic observation locator is out of bounds") semantic_support = semantic_supports[observation_index] if semantic_support.document != snapshot: raise E30MaterializationError( "replayed semantic support differs from immutable E29" ) original = _object(objects[observation_index], "fusion object") detector_score: float | None = _required_float(original, "score") selected_indices = semantic_support.occupied_source_indices candidate_indices = _semantic_candidate_indices( original, projected, fusion_profile, ) elif locator_kind == "geometry-only-cluster": cluster_index = _required_int(locator, "cluster_index") if not 0 <= cluster_index < len(geometry_supports): raise E30MaterializationError("geometry cluster locator is out of bounds") geometry_support = geometry_supports[cluster_index] detector_score = None if geometry_support.document != snapshot: raise E30MaterializationError( "replayed geometry cluster differs from immutable E29" ) selected_indices = geometry_support.occupied_source_indices candidate_indices = selected_indices else: raise E30MaterializationError("E30 locator kind is incompatible") if ( np.any(selected_indices < 0) or np.any(selected_indices >= points_map.shape[0]) or np.any(candidate_indices < 0) or np.any(candidate_indices >= points_map.shape[0]) or not set(int(value) for value in selected_indices).issubset( int(value) for value in candidate_indices ) ): raise E30MaterializationError("materialized point ownership is invalid") projected_rows = _bounded_projection_rows( projected, selected_indices=selected_indices, candidate_indices=candidate_indices, maximum_context_points=maximum_context_points, ) if projected is None: projected_indices = np.empty(0, dtype=np.int64) pixels = np.empty((0, 2), dtype=np.float32) depths = np.empty(0, dtype=np.float32) else: projected_indices = projected.source_indices[projected_rows].astype( np.int64, copy=False, ) pixels = projected.pixels_xy[projected_rows].astype(np.float32) depths = projected.depths_m[projected_rows].astype(np.float32) projected_candidate = np.isin( projected_indices, candidate_indices, assume_unique=False, ).astype(np.uint8) projected_selected = np.isin( projected_indices, selected_indices, assume_unique=False, ).astype(np.uint8) selected_indices = np.unique(selected_indices).astype(np.int64, copy=False) candidate_indices = np.unique(candidate_indices).astype(np.int64, copy=False) rejected_count = int( np.count_nonzero(~np.isin(candidate_indices, selected_indices)) ) arrays = _ItemArrays( selected_source_indices=selected_indices, selected_points_map_xyz_m=points_map[selected_indices].astype(np.float32), candidate_source_indices=candidate_indices, candidate_points_map_xyz_m=points_map[candidate_indices].astype(np.float32), projected_source_indices=projected_indices, projected_points_map_xyz_m=points_map[projected_indices].astype(np.float32), projected_pixels_xy=pixels, projected_depth_m=depths, projected_point_class=point_class[projected_indices].astype(np.uint8), projected_point_height_m=point_height[projected_indices].astype(np.float32), projected_candidate_mask=projected_candidate, projected_selected_mask=projected_selected, sensor_position_map_xyz_m=position, sensor_orientation_map_from_lidar_xyzw=orientation, ) metadata: dict[str, object] = { "frame_point_count": int(points_map.shape[0]), "camera_front_point_count": ( 0 if projected is None else projected.camera_front_point_count ), "projected_point_count": int(projected_indices.size), "candidate_point_count": int(candidate_indices.size), "selected_point_count": int(selected_indices.size), "rejected_candidate_point_count": rejected_count, "selected_and_candidate_lossless": True, "context_projection_bounded": True, "source_reprojection_required": False, "projection_width": int(projection_profile.width), "projection_height": int(projection_profile.height), "free_space_valid": False, "human_review_complete": False, "detector_score": detector_score, } return arrays, metadata def _engineering_triage( *, item: dict[str, Any], metadata: dict[str, object], camera_available: bool, ) -> dict[str, object]: """Route evidence without pretending that a rule is a semantic verdict.""" selected_count_value = metadata["selected_point_count"] if ( not isinstance(selected_count_value, int) or isinstance(selected_count_value, bool) or selected_count_value < 0 ): raise E30MaterializationError("selected point count is invalid") selected_count = selected_count_value detector_score = metadata.get("detector_score") stratum = _required_string(item, "stratum") locator = _required_object(item, "e29_locator") signals: list[str] = [] if selected_count <= 1: signals.append("sparse_selected_support") if isinstance(detector_score, float) and detector_score < 0.2: signals.append("low_detector_confidence") if locator.get("kind") == "geometry-only-cluster": signals.append("no_semantic_observation") if stratum == "unknown": signals.append("source_observation_not_current") return { "schema_version": "missioncore.e30-engineering-triage/v1", "provenance": "deterministic-evidence-readiness/v1", "state": ( "ready-for-ai-review" if camera_available else "blocked-camera-frame-unavailable" ), "attention": "elevated" if signals else "standard", "signals": signals, "semantic_verdict": None, "human_exception_required": None, } def _semantic_candidate_indices( item: dict[str, Any], projected: ProjectedPointCloud | None, profile: CameraGeometryFusionProfile, ) -> IntArray: bbox = item.get("bbox_xyxy") if projected is None or not isinstance(bbox, list) or len(bbox) != 4: return np.empty(0, dtype=np.int64) bounds = np.asarray(bbox, dtype=np.float64) if ( not np.isfinite(bounds).all() or bounds[2] <= bounds[0] or bounds[3] <= bounds[1] ): return np.empty(0, dtype=np.int64) width = float(bounds[2] - bounds[0]) height = float(bounds[3] - bounds[1]) inset = profile.bbox_inset_fraction inner = np.asarray( [ bounds[0] + width * inset, bounds[1] + height * inset, bounds[2] - width * inset, bounds[3] - height * inset, ], dtype=np.float64, ) pixels = projected.pixels_xy rows = np.flatnonzero( (pixels[:, 0] >= inner[0]) & (pixels[:, 0] <= inner[2]) & (pixels[:, 1] >= inner[1]) & (pixels[:, 1] <= inner[3]) ) return projected.source_indices[rows].astype(np.int64, copy=False) def _bounded_projection_rows( projected: ProjectedPointCloud | None, *, selected_indices: IntArray, candidate_indices: IntArray, maximum_context_points: int, ) -> IntArray: if projected is None: return np.empty(0, dtype=np.int64) required = np.isin( projected.source_indices, np.union1d(selected_indices, candidate_indices), ) required_rows = np.flatnonzero(required).astype(np.int64, copy=False) context_rows = np.flatnonzero(~required).astype(np.int64, copy=False) if context_rows.size > maximum_context_points: positions = np.linspace( 0, context_rows.size - 1, maximum_context_points, dtype=np.int64, ) context_rows = context_rows[positions] return np.sort(np.concatenate((required_rows, context_rows))).astype( np.int64, copy=False, ) def _read_review_pack(root: Path) -> _ReviewPack: candidate = root.expanduser() if candidate.is_symlink(): raise E30MaterializationError("E30 review pack must not be a symlink") candidate = candidate.resolve(strict=True) if not candidate.is_dir() or _REVIEW_PACK_ID.fullmatch(candidate.name) is None: raise E30MaterializationError("E30 review pack id is invalid") manifest_path = _regular_file(candidate, "manifest.json") manifest = _read_json(manifest_path, "E30 review manifest") identity = _required_object(manifest, "identity") identity_sha256 = _required_string(manifest, "identity_sha256") if ( manifest.get("schema_version") != "missioncore.e30-evidence-review-pack/v1" or manifest.get("result_id") != candidate.name or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256 or candidate.name != f"e30-review-pack-{identity_sha256}" or manifest.get("human_review_complete") is not False or manifest.get("lab_published") is not False ): raise E30MaterializationError("E30 review pack identity is invalid") _reject_authority(_required_object(manifest, "authority"), "E30 review pack") artifact = _unique_artifact(manifest, "review-items") items_path = _verified_relative_artifact(candidate, artifact) items: list[dict[str, Any]] = [] with items_path.open("r", encoding="utf-8") as stream: for expected_sequence, line in enumerate(stream): try: item = _object(json.loads(line), "E30 review item") except json.JSONDecodeError as exc: raise E30MaterializationError("E30 review item JSON is invalid") from exc item_id = _required_string(item, "item_id") if ( item.get("schema_version") != "missioncore.e30-evidence-review-item/v1" or _REVIEW_ITEM_ID.fullmatch(item_id) is None or _required_int(item, "sequence") != expected_sequence or _required_object(item, "review").get("state") != "unreviewed" ): raise E30MaterializationError("E30 review item is incompatible") _reject_authority( _required_object(item, "authority"), "E30 review item", ) items.append(item) if ( len(items) != _required_int(manifest, "selected_item_count") or len({item["item_id"] for item in items}) != len(items) ): raise E30MaterializationError("E30 review item count is inconsistent") return _ReviewPack( root=candidate, manifest=manifest, items=tuple(items), items_sha256=_required_string(artifact, "sha256"), ) def _review_items_by_frame( items: tuple[dict[str, Any], ...], ) -> dict[int, list[dict[str, Any]]]: grouped: dict[int, list[dict[str, Any]]] = {} for item in items: grouped.setdefault(_item_frame_index(item), []).append(item) return grouped def _item_frame_index(item: dict[str, Any]) -> int: binding = _required_object(item, "evidence_binding") frame_index = _required_int(binding, "frame_index") if frame_index < 0: raise E30MaterializationError("E30 frame index is invalid") return frame_index def _item_source_frame_index(item: dict[str, Any]) -> int: binding = _required_object(item, "evidence_binding") frame_index = _required_int(binding, "source_frame_index") if frame_index < 0: raise E30MaterializationError("E30 source frame index is invalid") return frame_index def _read_selected_fusion_frames( path: Path, *, selected_frame_indices: set[int], source: E10LidarFieldSource, ) -> dict[int, dict[str, Any]]: selected: dict[int, dict[str, Any]] = {} with path.open("r", encoding="utf-8") as stream: for frame_index, line in enumerate(stream): if frame_index not in selected_frame_indices: continue selected[frame_index] = _fusion_frame( line, expected_frame_index=frame_index, source=source, ) if set(selected) != selected_frame_indices: raise E30MaterializationError("source fusion frames are incomplete") return selected def _validate_bound_sources( source: E10LidarFieldSource, surface: K1LocalSurfaceV1, e29_identity: dict[str, Any], ) -> None: if ( e29_identity.get("source_pack_id") != source.pack_id or e29_identity.get("local_surface_model_id") != surface.model_id or surface.identity.get("source_pack_id") != source.pack_id or surface.identity.get("frame_count") != source.frame_count or surface.identity.get("point_count") != source.point_count or e29_identity.get("frame_count") != source.frame_count ): raise E30MaterializationError("E30 LiDAR/surface source binding is invalid") def _fusion_profile(identity: dict[str, Any]) -> CameraGeometryFusionProfile: value = _required_object(identity, "profile") profile = CameraGeometryFusionProfile( profile_id=_required_string(value, "profile_id"), bbox_inset_fraction=_required_float(value, "bbox_inset_fraction"), depth_cluster_minimum_gap_m=_required_float( value, "depth_cluster_minimum_gap_m", ), depth_cluster_gap_fraction=_required_float( value, "depth_cluster_gap_fraction", ), spatial_cluster_radius_m=_required_float( value, "spatial_cluster_radius_m", ), semantic_minimum_occupied_points=_required_int( value, "semantic_minimum_occupied_points", ), semantic_minimum_occupied_voxels=_required_int( value, "semantic_minimum_occupied_voxels", ), semantic_voxel_size_m=_required_float( value, "semantic_voxel_size_m", ), conflict_minimum_classified_points=_required_int( value, "conflict_minimum_classified_points", ), conflict_surface_fraction=_required_float( value, "conflict_surface_fraction", ), geometry_local_radius_m=_required_float( value, "geometry_local_radius_m", ), geometry_voxel_size_m=_required_float( value, "geometry_voxel_size_m", ), geometry_minimum_cluster_points=_required_int( value, "geometry_minimum_cluster_points", ), geometry_minimum_cluster_voxels=_required_int( value, "geometry_minimum_cluster_voxels", ), maximum_geometry_clusters_per_frame=_required_int( value, "maximum_geometry_clusters_per_frame", ), ) if profile.to_dict() != value: raise E30MaterializationError("E29 fusion profile cannot be reconstructed") return profile def _source_artifact_sha256(manifest: dict[str, Any]) -> str: artifact = _required_object(manifest, "artifact") return _required_sha256(artifact, "sha256") def _surface_artifact_sha256(manifest: dict[str, Any]) -> str: artifact = _unique_artifact(manifest, "local-surface") return _required_sha256(artifact, "sha256") def _read_existing_materialization( root: Path, identity: dict[str, object], ) -> dict[str, Any]: manifest = _read_json( _regular_file(root, E30_MATERIALIZATION_MANIFEST_NAME), "E30 materialization manifest", ) if ( manifest.get("schema_version") != E30_MATERIALIZATION_SCHEMA or manifest.get("identity") != identity or manifest.get("human_review_complete") is not False or manifest.get("lab_published") is not False ): raise E30MaterializationError("existing E30 materialization differs") _reject_authority( _required_object(manifest, "authority"), "E30 materialization", ) artifact = _unique_artifact(manifest, "materialized-items") _verified_relative_artifact(root, artifact) return manifest def _logical_arrays_sha256(arrays: dict[str, npt.NDArray[Any]]) -> str: digest = hashlib.sha256() for name in sorted(arrays): array = np.ascontiguousarray(arrays[name]) digest.update(name.encode()) digest.update(array.dtype.str.encode()) digest.update(_canonical_json(list(array.shape))) digest.update(array.tobytes(order="C")) return digest.hexdigest() def _child_directory(root: Path, child: str) -> Path: base = root.expanduser().absolute() if base.is_symlink(): raise E30MaterializationError("evidence root must not be a symlink") base = base.resolve(strict=True) candidate = base / child if candidate.is_symlink(): raise E30MaterializationError("evidence child must not be a symlink") candidate = candidate.resolve(strict=True) try: candidate.relative_to(base) except ValueError as exc: raise E30MaterializationError("evidence child escaped its root") from exc if not candidate.is_dir(): raise E30MaterializationError("evidence child must be a directory") return candidate def _regular_file(root: Path, relative: str) -> Path: candidate = root / relative if candidate.is_symlink(): raise E30MaterializationError("evidence artifact must not be a symlink") candidate = candidate.resolve(strict=True) try: candidate.relative_to(root) except ValueError as exc: raise E30MaterializationError("evidence artifact escaped its root") from exc if not candidate.is_file(): raise E30MaterializationError("evidence artifact must be a regular file") return candidate def _unique_artifact(manifest: dict[str, Any], role: str) -> dict[str, Any]: matches = [ _object(item, "artifact") for item in _required_array(manifest, "artifacts") if isinstance(item, dict) and item.get("role") == role ] if len(matches) != 1: raise E30MaterializationError(f"no unique {role} artifact") return matches[0] def _verified_relative_artifact( root: Path, artifact: dict[str, Any], ) -> Path: relative = _required_string(artifact, "path") if Path(relative).is_absolute() or ".." in Path(relative).parts: raise E30MaterializationError("artifact path is invalid") path = _regular_file(root, relative) if ( path.stat().st_size != _required_int(artifact, "byte_length") or _sha256_file(path) != _required_sha256(artifact, "sha256") ): raise E30MaterializationError("artifact content changed") return path def _artifact(role: str, path: Path, media_type: str) -> dict[str, object]: return { "role": role, "path": path.name, "media_type": media_type, "byte_length": path.stat().st_size, "sha256": _sha256_file(path), } def _reject_authority(authority: dict[str, Any], label: str) -> None: if ( authority.get("commands_enabled") is not False or authority.get("navigation_or_safety_accepted") is not False ): raise E30MaterializationError(f"{label} must remain diagnostic-only") def _read_json(path: Path, label: str) -> dict[str, Any]: try: return _object(json.loads(path.read_text(encoding="utf-8")), label) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise E30MaterializationError(f"{label} is invalid") from exc def _write_json(path: Path, value: dict[str, Any]) -> None: with path.open("x", encoding="utf-8") as stream: stream.write(_canonical_json(value).decode("utf-8")) stream.write("\n") def _canonical_json(value: object) -> bytes: return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False, ).encode() def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: while chunk := stream.read(1024 * 1024): digest.update(chunk) return digest.hexdigest() def _object(value: object, label: str) -> dict[str, Any]: if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): raise E30MaterializationError(f"{label} must be an object") return value def _required_object(document: dict[str, Any], key: str) -> dict[str, Any]: return _object(document.get(key), key) def _required_array(document: dict[str, Any], key: str) -> list[object]: value = document.get(key) if not isinstance(value, list): raise E30MaterializationError(f"{key} must be an array") return value def _required_string(document: dict[str, Any], key: str) -> str: value = document.get(key) if not isinstance(value, str) or not value: raise E30MaterializationError(f"{key} must be a nonempty string") return value def _required_sha256(document: dict[str, Any], key: str) -> str: value = _required_string(document, key) if _SHA256.fullmatch(value) is None: raise E30MaterializationError(f"{key} must be a SHA-256 digest") return value def _required_int(document: dict[str, Any], key: str) -> int: value = document.get(key) if not isinstance(value, int) or isinstance(value, bool): raise E30MaterializationError(f"{key} must be an integer") return value def _required_float(document: dict[str, Any], key: str) -> float: value = document.get(key) if not isinstance(value, int | float) or isinstance(value, bool): raise E30MaterializationError(f"{key} must be numeric") parsed = float(value) if not math.isfinite(parsed): raise E30MaterializationError(f"{key} must be finite") return parsed