feat(lidar): add RAVNOVES field review

This commit is contained in:
DCCONSTRUCTIONS
2026-07-25 10:26:07 +03:00
parent 2dfb34ef21
commit 3333e9ac0f
13 changed files with 2775 additions and 14 deletions
+30
View File
@@ -69,6 +69,22 @@ from .lidar_contract import (
lidar_readiness_document,
sensor_frame_xyzi,
)
from .lidar_field_review import (
E10_LIDAR_PACK_SCHEMA,
FIELD_REVIEW_ARRAYS_NAME,
FIELD_REVIEW_MANIFEST_NAME,
FIELD_REVIEW_REPORT_NAME,
LIDAR_FIELD_REVIEW_REPORT_SCHEMA,
LIDAR_FIELD_REVIEW_SCHEMA,
LIDAR_FIELD_REVIEW_WINDOW_SCHEMA,
MAX_FIELD_REVIEW_WINDOW_POINTS,
RAVNOVES00_CENTRAL_WINDOWS,
E10LidarFieldSource,
FieldReviewWindowSpec,
LidarFieldReviewV1,
build_lidar_field_review,
lidar_field_review_catalog_item,
)
from .lidar_ground import (
DEFAULT_GROUND_BENCHMARK_PROFILE,
LIDAR_GROUND_ANNOTATION_TEMPLATE_SCHEMA,
@@ -179,6 +195,9 @@ __all__ = [
"LIDAR_GROUND_BENCHMARK_REPORT_SCHEMA",
"LIDAR_GROUND_BENCHMARK_SCHEMA",
"LIDAR_GROUND_FRAME_SCHEMA",
"LIDAR_FIELD_REVIEW_REPORT_SCHEMA",
"LIDAR_FIELD_REVIEW_SCHEMA",
"LIDAR_FIELD_REVIEW_WINDOW_SCHEMA",
"LIDAR_EVIDENCE_PROFILE_SCHEMA",
"LIDAR_EQUIVALENCE_REPORT_SCHEMA",
"LIDAR_QUALITY_REPORT_SCHEMA",
@@ -262,6 +281,17 @@ __all__ = [
"build_lidar_ground_annotation_template",
"build_lidar_ground_benchmark",
"lidar_ground_frame_detail",
"E10_LIDAR_PACK_SCHEMA",
"FIELD_REVIEW_ARRAYS_NAME",
"FIELD_REVIEW_MANIFEST_NAME",
"FIELD_REVIEW_REPORT_NAME",
"MAX_FIELD_REVIEW_WINDOW_POINTS",
"RAVNOVES00_CENTRAL_WINDOWS",
"E10LidarFieldSource",
"FieldReviewWindowSpec",
"LidarFieldReviewV1",
"build_lidar_field_review",
"lidar_field_review_catalog_item",
"DetectionFrame",
"ObjectDetection",
"RecordedPerceptionOverlayError",
+891
View File
@@ -0,0 +1,891 @@
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
import numpy as np
import numpy.typing as npt
from k1link.device_plugins.xgrids_k1.analyze import (
CalibratedProjectionError,
map_points_to_lidar,
)
from .lidar_ground import (
GroundBenchmarkProfile,
GroundSegmenter,
LidarGroundError,
LocalPercentileGroundSegmenter,
)
LIDAR_FIELD_REVIEW_SCHEMA: Final = "missioncore.lidar-field-review/v1"
LIDAR_FIELD_REVIEW_REPORT_SCHEMA: Final = "missioncore.lidar-field-review-report/v1"
LIDAR_FIELD_REVIEW_WINDOW_SCHEMA: Final = "missioncore.lidar-field-review-window/v1"
E10_LIDAR_PACK_SCHEMA: Final = "missioncore.e10-lidar-replay-pack/v1"
FIELD_REVIEW_ARRAYS_NAME: Final = "field-review.npz"
FIELD_REVIEW_REPORT_NAME: Final = "field-review.json"
FIELD_REVIEW_MANIFEST_NAME: Final = "manifest.json"
MAX_FIELD_REVIEW_WINDOW_POINTS: Final = 80_000
_E10_PACK_ID = re.compile(r"^e10-lidar-pack-[a-f0-9]{64}$")
_FIELD_REVIEW_ID = re.compile(r"^lidar-field-review-[a-f0-9]{64}$")
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_SAFE_KEY = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
@dataclass(frozen=True, slots=True)
class FieldReviewWindowSpec:
key: str
label: str
start_seconds: float
end_seconds: float
preview_source_frame_index: int
def __post_init__(self) -> None:
if (
_SAFE_KEY.fullmatch(self.key) is None
or not self.label.strip()
or len(self.label) > 120
or not np.isfinite((self.start_seconds, self.end_seconds)).all()
or not 0 <= self.start_seconds < self.end_seconds
or self.preview_source_frame_index < 0
):
raise LidarGroundError("LiDAR field-review window is invalid")
def to_dict(self) -> dict[str, object]:
return {
"key": self.key,
"label": self.label,
"start_seconds": self.start_seconds,
"end_seconds": self.end_seconds,
"preview_source_frame_index": self.preview_source_frame_index,
}
RAVNOVES00_CENTRAL_WINDOWS: Final = (
FieldReviewWindowSpec(
key="intersection-facades",
label="Перекрёсток, дорога и фасады",
start_seconds=145.0,
end_seconds=151.0,
preview_source_frame_index=1120,
),
FieldReviewWindowSpec(
key="crossing-parked-vehicles",
label="Переход и припаркованные машины",
start_seconds=153.0,
end_seconds=159.0,
preview_source_frame_index=1200,
),
FieldReviewWindowSpec(
key="long-street",
label="Длинный фасад, тротуар и улица",
start_seconds=167.0,
end_seconds=175.0,
preview_source_frame_index=1350,
),
FieldReviewWindowSpec(
key="sidewalk-vehicles",
label="Тротуар, дома и автомобили",
start_seconds=180.0,
end_seconds=187.0,
preview_source_frame_index=1480,
),
FieldReviewWindowSpec(
key="street-vegetation",
label="Продолжение улицы и растительность",
start_seconds=188.0,
end_seconds=194.0,
preview_source_frame_index=1550,
),
)
class E10LidarFieldSource:
"""Strict reader for the immutable, intensity-free RAVNOVES00 E10 pack."""
def __init__(self, root: Path) -> None:
candidate = root.expanduser().absolute()
if candidate.is_symlink():
raise LidarGroundError("E10 LiDAR source cannot be a symlink")
self.root = candidate.resolve(strict=True)
if not self.root.is_dir() or _E10_PACK_ID.fullmatch(self.root.name) is None:
raise LidarGroundError("E10 LiDAR source id is invalid")
self.manifest = _read_json(self.root / "manifest.json")
self.identity = _object(self.manifest.get("identity"), "E10 LiDAR identity")
identity_sha256 = self.manifest.get("identity_sha256")
artifact = _object(self.manifest.get("artifact"), "E10 LiDAR artifact")
artifact_path = artifact.get("path")
if (
self.manifest.get("schema_version") != E10_LIDAR_PACK_SCHEMA
or self.identity.get("schema_version") != E10_LIDAR_PACK_SCHEMA
or not isinstance(identity_sha256, str)
or _SHA256.fullmatch(identity_sha256) is None
or hashlib.sha256(_canonical_json(self.identity)).hexdigest() != identity_sha256
or self.root.name != f"e10-lidar-pack-{identity_sha256}"
or self.manifest.get("pack_id") != self.root.name
or artifact_path != "lidar-pack.npz"
):
raise LidarGroundError("E10 LiDAR source identity is invalid")
arrays_path = self.root / artifact_path
if (
arrays_path.is_symlink()
or not arrays_path.is_file()
or arrays_path.stat().st_size != artifact.get("byte_length")
or _sha256(arrays_path) != artifact.get("sha256")
):
raise LidarGroundError("E10 LiDAR source artifact is invalid")
self.arrays = np.load(arrays_path, allow_pickle=False)
try:
self._validate_arrays()
except BaseException:
self.close()
raise
self.pack_id = self.root.name
@property
def frame_count(self) -> int:
return int(self.identity["frame_count"])
@property
def point_count(self) -> int:
return int(self.identity["point_count"])
def close(self) -> None:
self.arrays.close()
def _validate_arrays(self) -> None:
required = {
"frame_indices",
"source_frame_indices",
"session_seconds",
"sample_available",
"cloud_offsets",
"cloud_points_map",
"pose_positions_map",
"pose_quaternions_map_from_lidar",
"lidar_camera_delta_ms",
"pose_point_delta_ms",
"intrinsic_fx_fy_cx_cy",
"distortion_kb4",
"t_camera_from_lidar",
}
frame_count = _nonnegative_int(self.identity.get("frame_count"), "E10 frame count")
point_count = _nonnegative_int(self.identity.get("point_count"), "E10 point count")
available = self.arrays["sample_available"]
offsets = self.arrays["cloud_offsets"]
points = self.arrays["cloud_points_map"]
positions = self.arrays["pose_positions_map"]
quaternions = self.arrays["pose_quaternions_map_from_lidar"]
if (
set(self.arrays.files) != required
or self.arrays["frame_indices"].shape != (frame_count,)
or self.arrays["source_frame_indices"].shape != (frame_count,)
or self.arrays["session_seconds"].shape != (frame_count,)
or available.shape != (frame_count,)
or offsets.shape != (frame_count + 1,)
or points.shape != (point_count, 3)
or positions.shape != (frame_count, 3)
or quaternions.shape != (frame_count, 4)
or self.arrays["frame_indices"].dtype != np.dtype("<i8")
or self.arrays["source_frame_indices"].dtype != np.dtype("<i8")
or self.arrays["session_seconds"].dtype != np.dtype("<f8")
or available.dtype != np.dtype("?")
or offsets.dtype != np.dtype("<i8")
or points.dtype != np.dtype("<f4")
or positions.dtype != np.dtype("<f8")
or quaternions.dtype != np.dtype("<f8")
or not np.array_equal(
self.arrays["frame_indices"],
np.arange(frame_count, dtype=np.int64),
)
or not np.all(np.diff(self.arrays["source_frame_indices"]) > 0)
or not np.isfinite(self.arrays["session_seconds"]).all()
or not np.all(np.diff(self.arrays["session_seconds"]) > 0)
or offsets[0] != 0
or offsets[-1] != point_count
or np.any(np.diff(offsets) < 0)
or not np.isfinite(points).all()
or int(np.count_nonzero(available)) != self.identity.get("available_lidar_frames")
):
raise LidarGroundError("E10 LiDAR source arrays are invalid")
counts = np.diff(offsets)
if (
np.any(counts[available] <= 0)
or np.any(counts[~available] != 0)
or not np.isfinite(positions[available]).all()
or not np.isfinite(quaternions[available]).all()
):
raise LidarGroundError("E10 LiDAR source availability is invalid")
class LidarFieldReviewV1:
"""Strict reader for accumulated, path-free LiDAR field-review evidence."""
def __init__(self, root: Path) -> None:
candidate = root.expanduser().absolute()
if candidate.is_symlink():
raise LidarGroundError("LiDAR field review cannot be a symlink")
self.root = candidate.resolve(strict=True)
if not self.root.is_dir() or _FIELD_REVIEW_ID.fullmatch(self.root.name) is None:
raise LidarGroundError("LiDAR field-review id is invalid")
self.manifest = _read_json(self.root / FIELD_REVIEW_MANIFEST_NAME)
self.identity = _object(self.manifest.get("identity"), "field-review identity")
identity_sha256 = self.manifest.get("identity_sha256")
if (
self.manifest.get("schema_version") != LIDAR_FIELD_REVIEW_SCHEMA
or self.identity.get("schema_version") != LIDAR_FIELD_REVIEW_SCHEMA
or not isinstance(identity_sha256, str)
or _SHA256.fullmatch(identity_sha256) is None
or hashlib.sha256(_canonical_json(self.identity)).hexdigest() != identity_sha256
or self.root.name != f"lidar-field-review-{identity_sha256}"
or self.manifest.get("review_id") != self.root.name
):
raise LidarGroundError("LiDAR field-review identity is invalid")
artifacts = _validate_artifacts(self.root, self.manifest.get("artifacts"))
self.arrays = np.load(artifacts["field-review"], allow_pickle=False)
self.report = _read_json(artifacts["field-review-report"])
self.preview_paths = {
role.removeprefix("preview-"): path
for role, path in artifacts.items()
if role.startswith("preview-")
}
try:
self._validate()
except BaseException:
self.close()
raise
self.review_id = self.root.name
def close(self) -> None:
self.arrays.close()
def _validate(self) -> None:
windows = _list(self.report.get("windows"), "field-review windows")
parsed_windows = [
_object(item, f"field-review window {index}") for index, item in enumerate(windows)
]
source = _object(self.report.get("source"), "field-review source")
decision = _object(self.report.get("decision"), "field-review decision")
authority = _object(self.report.get("authority"), "field-review authority")
offsets = self.arrays["window_offsets"]
points = self.arrays["points_xyz_map"]
point_count = _nonnegative_int(
self.identity.get("display_point_count"),
"field-review point count",
)
required = {
"window_offsets",
"points_xyz_map",
"current_ground",
"current_assigned",
"candidate_ground",
"candidate_assigned",
}
if (
self.report.get("schema_version") != LIDAR_FIELD_REVIEW_REPORT_SCHEMA
or self.report.get("review_id") != self.root.name
or self.report.get("status") != "diagnostic-only"
or self.report.get("ground_truth") is not False
or source.get("representation") != "legacy-e10-vendor-map-with-pose"
or source.get("intensity_available") is not False
or source.get("raw_scan_accepted") is not False
or decision.get("status") != "visual-review-only"
or decision.get("production_promotion") is not False
or authority.get("commands_enabled") is not False
or authority.get("navigation_or_safety_accepted") is not False
or len(windows) != self.identity.get("window_count")
or set(self.preview_paths) != {str(item.get("key")) for item in parsed_windows}
or set(self.arrays.files) != required
or offsets.shape != (len(windows) + 1,)
or offsets.dtype != np.dtype("<i8")
or offsets[0] != 0
or offsets[-1] != point_count
or np.any(np.diff(offsets) <= 0)
or points.shape != (point_count, 3)
or points.dtype != np.dtype("<f4")
or not np.isfinite(points).all()
or _logical_sha256(self.arrays) != self.identity.get("logical_content_sha256")
):
raise LidarGroundError("LiDAR field-review content is invalid")
for name in (
"current_ground",
"current_assigned",
"candidate_ground",
"candidate_assigned",
):
value = self.arrays[name]
if value.shape != (point_count,) or value.dtype != np.dtype("u1") or np.any(value > 1):
raise LidarGroundError("LiDAR field-review mask is invalid")
for index, window in enumerate(parsed_windows):
start = int(offsets[index])
end = int(offsets[index + 1])
if (
window.get("index") != index
or _SAFE_KEY.fullmatch(str(window.get("key"))) is None
or window.get("display_point_count") != end - start
or not 0 < end - start <= MAX_FIELD_REVIEW_WINDOW_POINTS
or _nonnegative_int(
window.get("source_lidar_samples"),
"field-review source samples",
)
< 1
or _nonnegative_int(
window.get("source_point_count"),
"field-review source points",
)
< end - start
or not isinstance(window.get("label"), str)
or not window["label"].strip()
):
raise LidarGroundError("LiDAR field-review window is invalid")
def window_detail(self, window_index: int) -> dict[str, object]:
windows = _list(self.report["windows"], "field-review windows")
if not 0 <= window_index < len(windows):
raise IndexError(window_index)
window = _object(windows[window_index], "field-review window")
offsets = self.arrays["window_offsets"]
start = int(offsets[window_index])
end = int(offsets[window_index + 1])
current_ground = self.arrays["current_ground"][start:end]
candidate_ground = self.arrays["candidate_ground"][start:end]
disagreement = (current_ground != candidate_ground).astype(np.uint8)
return {
"schema_version": LIDAR_FIELD_REVIEW_WINDOW_SCHEMA,
"review_id": self.review_id,
"display_name": self.report["display_name"],
"session_id": self.report["session_id"],
"source_pack_id": self.report["source_pack_id"],
"window_index": window_index,
"window_count": len(windows),
"window": window,
"point_count": end - start,
"coordinate_frame": "map",
"distance_unit": "m",
"intensity": {
"available": False,
"reason": "E10 derivative did not retain rgbi/intensity",
},
"points_xyz_m": self.arrays["points_xyz_map"][start:end].astype(np.float64).tolist(),
"masks": {
"current_ground": current_ground.astype(np.int64).tolist(),
"current_assigned": self.arrays["current_assigned"][start:end]
.astype(np.int64)
.tolist(),
"candidate_ground": candidate_ground.astype(np.int64).tolist(),
"candidate_assigned": self.arrays["candidate_assigned"][start:end]
.astype(np.int64)
.tolist(),
"disagreement": disagreement.astype(np.int64).tolist(),
},
"counts": {
"current_ground": int(np.count_nonzero(current_ground)),
"candidate_ground": int(np.count_nonzero(candidate_ground)),
"disagreement": int(np.count_nonzero(disagreement)),
},
"preview_url": (
f"/api/v1/lidar/field-reviews/{self.review_id}/windows/{window_index}/preview"
),
"access": "read-only",
"ground_truth": False,
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
def build_lidar_field_review(
source: E10LidarFieldSource,
output_root: Path,
*,
patchwork: GroundSegmenter,
profile: GroundBenchmarkProfile,
preview_paths: Mapping[str, Path],
windows: Sequence[FieldReviewWindowSpec] = RAVNOVES00_CENTRAL_WINDOWS,
display_name: str = "RAVNOVES00 · центральный городской интервал",
default_window_index: int = 2,
maximum_display_points: int = MAX_FIELD_REVIEW_WINDOW_POINTS,
) -> Path:
"""Build accumulated field windows without upgrading legacy input evidence."""
if (
not windows
or not 0 <= default_window_index < len(windows)
or not 1 <= maximum_display_points <= MAX_FIELD_REVIEW_WINDOW_POINTS
or set(preview_paths) != {window.key for window in windows}
):
raise LidarGroundError("LiDAR field-review build configuration is invalid")
times = source.arrays["session_seconds"]
available = source.arrays["sample_available"]
source_offsets = source.arrays["cloud_offsets"]
points_map = source.arrays["cloud_points_map"]
positions = source.arrays["pose_positions_map"]
quaternions = source.arrays["pose_quaternions_map_from_lidar"]
source_frame_indices = source.arrays["source_frame_indices"]
current = LocalPercentileGroundSegmenter(profile)
window_arrays: list[dict[str, npt.NDArray[Any]]] = []
window_reports: list[dict[str, object]] = []
current_latency: list[float] = []
candidate_latency: list[float] = []
current_fraction: list[float] = []
candidate_fraction: list[float] = []
disagreement_fraction: list[float] = []
algorithm_iou: list[float] = []
selected_source_point_count = 0
for window_index, spec in enumerate(windows):
source_rows = np.flatnonzero(
available & (times >= spec.start_seconds) & (times <= spec.end_seconds)
)
if source_rows.size == 0:
raise LidarGroundError("LiDAR field-review window has no source samples")
collected_points: list[npt.NDArray[np.float32]] = []
collected_current: list[npt.NDArray[np.uint8]] = []
collected_current_assigned: list[npt.NDArray[np.uint8]] = []
collected_candidate: list[npt.NDArray[np.uint8]] = []
collected_candidate_assigned: list[npt.NDArray[np.uint8]] = []
source_point_count = 0
for source_row in source_rows:
start = int(source_offsets[source_row])
end = int(source_offsets[source_row + 1])
cloud = np.asarray(points_map[start:end], dtype=np.float32)
source_point_count += cloud.shape[0]
current_input = np.zeros((cloud.shape[0], 4), dtype=np.float32)
current_input[:, :3] = cloud
current_result = current.segment(current_input)
try:
position = positions[source_row]
orientation = quaternions[source_row]
points_sensor = map_points_to_lidar(
cloud,
position_map_xyz=(
float(position[0]),
float(position[1]),
float(position[2]),
),
orientation_map_from_lidar_xyzw=(
float(orientation[0]),
float(orientation[1]),
float(orientation[2]),
float(orientation[3]),
),
)
except CalibratedProjectionError as exc:
raise LidarGroundError("LiDAR field-review pose conversion failed") from exc
candidate_input = np.zeros((cloud.shape[0], 4), dtype=np.float32)
candidate_input[:, :3] = points_sensor.astype(np.float32)
if profile.patchwork_map_vertical_origin_offset_m:
candidate_input[:, 2] -= profile.patchwork_map_vertical_origin_offset_m
candidate_result = patchwork.segment(candidate_input)
_segmentation(
current_result.ground_mask,
current_result.assigned_mask,
cloud.shape[0],
"current",
)
_segmentation(
candidate_result.ground_mask,
candidate_result.assigned_mask,
cloud.shape[0],
"candidate",
)
collected_points.append(cloud)
collected_current.append(current_result.ground_mask.astype(np.uint8))
collected_current_assigned.append(current_result.assigned_mask.astype(np.uint8))
collected_candidate.append(candidate_result.ground_mask.astype(np.uint8))
collected_candidate_assigned.append(candidate_result.assigned_mask.astype(np.uint8))
current_latency.append(current_result.latency_ms)
candidate_latency.append(candidate_result.latency_ms)
current_fraction.append(float(np.mean(current_result.ground_mask)))
candidate_fraction.append(float(np.mean(candidate_result.ground_mask)))
disagreement_fraction.append(
float(np.mean(current_result.ground_mask != candidate_result.ground_mask))
)
intersection = int(
np.count_nonzero(current_result.ground_mask & candidate_result.ground_mask)
)
union = int(np.count_nonzero(current_result.ground_mask | candidate_result.ground_mask))
algorithm_iou.append(float(intersection / union) if union else 1.0)
combined_points = np.concatenate(collected_points)
combined_current = np.concatenate(collected_current)
combined_current_assigned = np.concatenate(collected_current_assigned)
combined_candidate = np.concatenate(collected_candidate)
combined_candidate_assigned = np.concatenate(collected_candidate_assigned)
selected_source_point_count += source_point_count
selected = _uniform_indices(combined_points.shape[0], maximum_display_points)
displayed = {
"points_xyz_map": combined_points[selected].astype("<f4"),
"current_ground": combined_current[selected].astype("u1"),
"current_assigned": combined_current_assigned[selected].astype("u1"),
"candidate_ground": combined_candidate[selected].astype("u1"),
"candidate_assigned": combined_candidate_assigned[selected].astype("u1"),
}
window_arrays.append(displayed)
preview_row = int(
np.argmin(
np.abs(source_frame_indices.astype(np.int64) - spec.preview_source_frame_index)
)
)
window_reports.append(
{
"index": window_index,
"key": spec.key,
"label": spec.label,
"start_seconds": spec.start_seconds,
"end_seconds": spec.end_seconds,
"midpoint_seconds": (spec.start_seconds + spec.end_seconds) / 2,
"source_lidar_samples": int(source_rows.shape[0]),
"source_point_count": source_point_count,
"display_point_count": int(selected.shape[0]),
"source_frame_start": int(source_frame_indices[source_rows[0]]),
"source_frame_end": int(source_frame_indices[source_rows[-1]]),
"preview_source_frame_index": int(source_frame_indices[preview_row]),
"preview_session_seconds": float(times[preview_row]),
}
)
offsets = np.concatenate(
(
np.asarray([0], dtype="<i8"),
np.cumsum(
[item["points_xyz_map"].shape[0] for item in window_arrays],
dtype=np.int64,
),
)
).astype("<i8")
arrays: dict[str, npt.NDArray[Any]] = {
"window_offsets": offsets,
"points_xyz_map": np.concatenate([item["points_xyz_map"] for item in window_arrays]).astype(
"<f4"
),
"current_ground": np.concatenate([item["current_ground"] for item in window_arrays]).astype(
"u1"
),
"current_assigned": np.concatenate(
[item["current_assigned"] for item in window_arrays]
).astype("u1"),
"candidate_ground": np.concatenate(
[item["candidate_ground"] for item in window_arrays]
).astype("u1"),
"candidate_assigned": np.concatenate(
[item["candidate_assigned"] for item in window_arrays]
).astype("u1"),
}
logical_content_sha256 = _logical_sha256(arrays)
identity = {
"schema_version": LIDAR_FIELD_REVIEW_SCHEMA,
"source_pack_id": source.pack_id,
"source_pack_identity_sha256": source.manifest["identity_sha256"],
"source_artifact_sha256": source.manifest["artifact"]["sha256"],
"session_id": source.identity["session_id"],
"display_name": display_name,
"windows": [window.to_dict() for window in windows],
"window_count": len(windows),
"default_window_index": default_window_index,
"source_point_count": selected_source_point_count,
"display_point_count": int(arrays["points_xyz_map"].shape[0]),
"sampling": {
"method": "uniform-point-index-per-window",
"maximum_points_per_window": maximum_display_points,
},
"profile": profile.to_dict(),
"providers": {
"current": dict(current.identity),
"candidate": dict(patchwork.identity),
},
"logical_content_sha256": logical_content_sha256,
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
"ground_truth": False,
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
review_id = f"lidar-field-review-{identity_sha256}"
output_parent = output_root.expanduser().resolve()
output_parent.mkdir(mode=0o700, parents=True, exist_ok=True)
output = output_parent / review_id
if output.exists():
existing = LidarFieldReviewV1(output)
existing.close()
return output
report = {
"schema_version": LIDAR_FIELD_REVIEW_REPORT_SCHEMA,
"review_id": review_id,
"display_name": display_name,
"session_id": source.identity["session_id"],
"source_pack_id": source.pack_id,
"status": "diagnostic-only",
"source": {
"timeline_start_seconds": source.identity["timeline_start_seconds"],
"timeline_end_seconds": source.identity["timeline_end_seconds"],
"available_lidar_frames": source.identity["available_lidar_frames"],
"point_count": source.identity["point_count"],
"representation": "legacy-e10-vendor-map-with-pose",
"intensity_available": False,
"raw_scan_accepted": False,
},
"selection": {
"purpose": "operator-readable-central-urban-field-review",
"default_window_index": default_window_index,
"accumulation": "per-source-frame masks accumulated in map frame",
"sampling": identity["sampling"],
},
"windows": window_reports,
"metrics": {
"source_samples": len(current_latency),
"current": {
"provider": dict(current.identity),
"ground_fraction": _distribution(current_fraction),
"latency_ms": _distribution(current_latency),
},
"candidate": {
"provider": dict(patchwork.identity),
"ground_fraction": _distribution(candidate_fraction),
"latency_ms": _distribution(candidate_latency),
},
"comparison": {
"algorithm_to_algorithm_ground_iou": _distribution(algorithm_iou),
"ground_disagreement_fraction": _distribution(disagreement_fraction),
"is_accuracy_metric": False,
},
},
"decision": {
"status": "visual-review-only",
"production_promotion": False,
"reasons": [
"legacy E10 derivative does not retain intensity",
"source remains a vendor-mapped LIO product",
"independent ground labels are missing",
],
},
"ground_truth": False,
"authority": identity["authority"],
}
staging = output_parent / f".{review_id}.{os.getpid()}.incomplete"
staging.mkdir(mode=0o700, exist_ok=False)
try:
arrays_path = staging / FIELD_REVIEW_ARRAYS_NAME
np.savez_compressed(arrays_path, **arrays) # type: ignore[arg-type]
report_path = staging / FIELD_REVIEW_REPORT_NAME
_write_json(report_path, report)
artifacts = [
_artifact("field-review", arrays_path, "application/x-npz"),
_artifact("field-review-report", report_path, "application/json"),
]
for spec in windows:
preview_source = preview_paths[spec.key].expanduser().resolve(strict=True)
if not preview_source.is_file() or preview_source.is_symlink():
raise LidarGroundError("LiDAR field-review preview is invalid")
preview_target = staging / f"preview-{spec.key}.jpg"
shutil.copy2(preview_source, preview_target)
artifacts.append(
_artifact(
f"preview-{spec.key}",
preview_target,
"image/jpeg",
)
)
manifest = {
"schema_version": LIDAR_FIELD_REVIEW_SCHEMA,
"review_id": review_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"classification": "private-derived-lidar-diagnostic",
"ground_truth": False,
"artifacts": artifacts,
}
_write_json(staging / FIELD_REVIEW_MANIFEST_NAME, manifest)
os.replace(staging, output)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
validation = LidarFieldReviewV1(output)
validation.close()
return output
def lidar_field_review_catalog_item(review: LidarFieldReviewV1) -> dict[str, object]:
report = review.report
metrics = _object(report["metrics"], "field-review metrics")
return {
"review_id": review.review_id,
"display_name": report["display_name"],
"session_id": report["session_id"],
"source_pack_id": report["source_pack_id"],
"status": report["status"],
"source": report["source"],
"selection": report["selection"],
"windows": report["windows"],
"metrics": metrics,
"decision": report["decision"],
"created_at_utc": review.manifest.get("created_at_utc"),
"ground_truth": False,
"authority": report["authority"],
}
def _uniform_indices(count: int, maximum: int) -> npt.NDArray[np.int64]:
if count <= maximum:
return np.arange(count, dtype=np.int64)
return np.linspace(0, count - 1, maximum, dtype=np.int64)
def _segmentation(
ground_mask: npt.NDArray[np.bool_],
assigned_mask: npt.NDArray[np.bool_],
point_count: int,
label: str,
) -> None:
if (
ground_mask.shape != (point_count,)
or ground_mask.dtype != np.dtype("?")
or assigned_mask.shape != (point_count,)
or assigned_mask.dtype != np.dtype("?")
or np.any(ground_mask & ~assigned_mask)
):
raise LidarGroundError(f"LiDAR field-review {label} mask is invalid")
def _distribution(values: Sequence[float]) -> dict[str, float | int]:
array = np.asarray(values, dtype=np.float64)
if array.size == 0 or not np.isfinite(array).all():
raise LidarGroundError("LiDAR field-review distribution is invalid")
return {
"sample_count": int(array.shape[0]),
"minimum": float(np.min(array)),
"mean": float(np.mean(array)),
"p50": float(np.percentile(array, 50)),
"p95": float(np.percentile(array, 95)),
"maximum": float(np.max(array)),
}
def _logical_sha256(arrays: Mapping[str, npt.NDArray[Any]]) -> str:
digest = hashlib.sha256()
for name in sorted(arrays):
array = np.ascontiguousarray(arrays[name])
digest.update(name.encode())
digest.update(array.dtype.str.encode())
digest.update(_canonical_json(list(array.shape)))
digest.update(memoryview(array).cast("B"))
return digest.hexdigest()
def _validate_artifacts(root: Path, value: object) -> dict[str, Path]:
artifacts = _list(value, "field-review artifacts")
resolved: dict[str, Path] = {}
for value in artifacts:
item = _object(value, "field-review artifact")
role = item.get("role")
relative = item.get("path")
if (
not isinstance(role, str)
or role in resolved
or not isinstance(relative, str)
or Path(relative).name != relative
or not isinstance(item.get("byte_length"), int)
or isinstance(item.get("byte_length"), bool)
or item["byte_length"] < 1
or _SHA256.fullmatch(str(item.get("sha256"))) is None
):
raise LidarGroundError("LiDAR field-review artifact descriptor is invalid")
path = root / relative
if (
path.is_symlink()
or not path.is_file()
or path.stat().st_size != item["byte_length"]
or _sha256(path) != item["sha256"]
):
raise LidarGroundError("LiDAR field-review artifact is invalid")
resolved[role] = path
if "field-review" not in resolved or "field-review-report" not in resolved:
raise LidarGroundError("LiDAR field-review artifacts are incomplete")
return resolved
def _artifact(role: str, path: Path, media_type: str) -> dict[str, object]:
return {
"role": role,
"path": path.name,
"media_type": media_type,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _read_json(path: Path) -> dict[str, Any]:
if path.is_symlink() or not path.is_file() or path.stat().st_size > 4_000_000:
raise LidarGroundError("LiDAR field-review JSON artifact is invalid")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
raise LidarGroundError("LiDAR field-review JSON artifact is unreadable") from exc
return _object(value, "LiDAR field-review JSON")
def _write_json(path: Path, value: Mapping[str, object]) -> None:
path.write_bytes(
json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
indent=2,
allow_nan=False,
).encode()
+ b"\n"
)
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode()
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
raise LidarGroundError(f"{label} must be an object")
return value
def _list(value: object, label: str) -> list[Any]:
if not isinstance(value, list):
raise LidarGroundError(f"{label} must be a list")
return value
def _nonnegative_int(value: object, label: str) -> int:
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
raise LidarGroundError(f"{label} must be a non-negative integer")
return value
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+146 -1
View File
@@ -6,13 +6,15 @@ from collections.abc import Callable
from pathlib import Path
from typing import Any, Final
from fastapi import APIRouter, HTTPException, Query
from fastapi import APIRouter, HTTPException, Query, Response
from k1link.compute import (
LidarFieldReviewV1,
LidarGroundBenchmarkV1,
LidarGroundError,
LidarReplayError,
LidarReplayPackV2,
lidar_field_review_catalog_item,
lidar_ground_benchmark_catalog_item,
lidar_ground_frame_detail,
lidar_pack_catalog_item,
@@ -21,8 +23,10 @@ from k1link.compute import (
LIDAR_CATALOG_SCHEMA: Final = "missioncore.lidar-replay-pack-catalog/v1"
LIDAR_GROUND_CATALOG_SCHEMA: Final = "missioncore.lidar-ground-benchmark-catalog/v1"
LIDAR_FIELD_REVIEW_CATALOG_SCHEMA: Final = "missioncore.lidar-field-review-catalog/v1"
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
_BENCHMARK_ID = re.compile(r"^ground-benchmark-[a-f0-9]{64}$")
_FIELD_REVIEW_ID = re.compile(r"^lidar-field-review-[a-f0-9]{64}$")
RootProvider = Callable[[], Path | None]
@@ -36,10 +40,16 @@ def configured_lidar_ground_root() -> Path | None:
return Path(value).expanduser().absolute() if value else None
def configured_lidar_field_review_root() -> Path | None:
value = os.environ.get("MISSIONCORE_LIDAR_FIELD_REVIEW_ROOT", "").strip()
return Path(value).expanduser().absolute() if value else None
def build_lidar_router(
*,
root_provider: RootProvider = configured_lidar_replay_root,
ground_root_provider: RootProvider = configured_lidar_ground_root,
field_review_root_provider: RootProvider = configured_lidar_field_review_root,
) -> APIRouter:
router = APIRouter(prefix="/api/v1/lidar", tags=["lidar"])
@@ -269,4 +279,139 @@ def build_lidar_router(
detail="LiDAR ground frame не прошёл проверку целостности",
) from exc
@router.get("/field-reviews")
def list_lidar_field_reviews(
limit: int = Query(default=10, ge=1, le=50),
) -> dict[str, Any]:
root = field_review_root_provider()
if root is None or not root.is_dir():
return {
"schema_version": LIDAR_FIELD_REVIEW_CATALOG_SCHEMA,
"configured": root is not None,
"items": [],
"valid_total": 0,
"invalid_total": 0,
"access": "read-only",
}
items: list[dict[str, object]] = []
invalid_total = 0
candidates = sorted(
(
candidate
for candidate in root.iterdir()
if candidate.is_dir() and _FIELD_REVIEW_ID.fullmatch(candidate.name) is not None
),
key=lambda candidate: candidate.stat().st_mtime_ns,
reverse=True,
)
for candidate in candidates:
try:
review = LidarFieldReviewV1(candidate)
try:
items.append(lidar_field_review_catalog_item(review))
finally:
review.close()
except (LidarGroundError, OSError):
invalid_total += 1
return {
"schema_version": LIDAR_FIELD_REVIEW_CATALOG_SCHEMA,
"configured": True,
"items": items[:limit],
"valid_total": len(items),
"invalid_total": invalid_total,
"access": "read-only",
}
@router.get("/field-reviews/{review_id}/windows/{window_index}")
def get_lidar_field_review_window(
review_id: str,
window_index: int,
) -> dict[str, object]:
if _FIELD_REVIEW_ID.fullmatch(review_id) is None or window_index < 0:
raise HTTPException(
status_code=404,
detail="LiDAR field-review window не найден",
)
root = field_review_root_provider()
if root is None or not root.is_dir():
raise HTTPException(
status_code=503,
detail="LiDAR field-review storage не настроен",
)
candidate = root / review_id
if not candidate.is_dir():
raise HTTPException(
status_code=404,
detail="LiDAR field review не найден",
)
try:
review = LidarFieldReviewV1(candidate)
try:
return review.window_detail(window_index)
finally:
review.close()
except IndexError as exc:
raise HTTPException(
status_code=404,
detail="LiDAR field-review window не найден",
) from exc
except (LidarGroundError, OSError) as exc:
raise HTTPException(
status_code=409,
detail="LiDAR field review не прошёл проверку целостности",
) from exc
@router.get("/field-reviews/{review_id}/windows/{window_index}/preview")
def get_lidar_field_review_preview(
review_id: str,
window_index: int,
) -> Response:
if _FIELD_REVIEW_ID.fullmatch(review_id) is None or window_index < 0:
raise HTTPException(
status_code=404,
detail="LiDAR field-review preview не найден",
)
root = field_review_root_provider()
if root is None or not root.is_dir():
raise HTTPException(
status_code=503,
detail="LiDAR field-review storage не настроен",
)
candidate = root / review_id
if not candidate.is_dir():
raise HTTPException(
status_code=404,
detail="LiDAR field review не найден",
)
try:
review = LidarFieldReviewV1(candidate)
try:
windows = review.report.get("windows")
if not isinstance(windows, list) or not 0 <= window_index < len(windows):
raise IndexError(window_index)
window = windows[window_index]
if not isinstance(window, dict) or not isinstance(window.get("key"), str):
raise LidarGroundError("LiDAR field-review preview key is invalid")
preview = review.preview_paths.get(window["key"])
if preview is None:
raise LidarGroundError("LiDAR field-review preview is missing")
content = preview.read_bytes()
finally:
review.close()
except IndexError as exc:
raise HTTPException(
status_code=404,
detail="LiDAR field-review preview не найден",
) from exc
except (LidarGroundError, OSError) as exc:
raise HTTPException(
status_code=409,
detail="LiDAR field-review preview не прошёл проверку",
) from exc
return Response(
content=content,
media_type="image/jpeg",
headers={"Cache-Control": "private, max-age=31536000, immutable"},
)
return router