354 lines
11 KiB
Python
354 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import math
|
|
import os
|
|
import stat
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import TypedDict
|
|
|
|
from k1link.artifacts import utc_now_iso
|
|
from k1link.mqtt import DEFAULT_MAX_MESSAGE_BYTES, iter_capture_frames
|
|
from k1link.protocol import (
|
|
DecodeLimits,
|
|
StreamDecodeError,
|
|
decode_lio_pcl,
|
|
decode_lio_pose,
|
|
)
|
|
|
|
LIO_PCL_TOPIC = "lixel/application/report/lio_pcl"
|
|
LIO_POSE_TOPIC = "lixel/application/report/lio_pose"
|
|
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES = DecodeLimits().max_mqtt_payload_bytes
|
|
MAX_STREAM_SUMMARY_PAYLOAD_BYTES = DEFAULT_MAX_MESSAGE_BYTES
|
|
_HASH_CHUNK_BYTES = 1024 * 1024
|
|
|
|
|
|
class IntRangeSummary(TypedDict):
|
|
min: int | None
|
|
max: int | None
|
|
|
|
|
|
class ScalerSummary(TypedDict):
|
|
min: int | None
|
|
max: int | None
|
|
constant: bool | None
|
|
|
|
|
|
class ByteSummary(TypedDict):
|
|
total: int
|
|
per_frame: IntRangeSummary
|
|
|
|
|
|
class PointSummary(TypedDict):
|
|
total: int
|
|
per_frame: IntRangeSummary
|
|
|
|
|
|
class SourceSummary(TypedDict):
|
|
bytes: int
|
|
sha256: str
|
|
|
|
|
|
class LimitSummary(TypedDict):
|
|
max_payload_bytes: int
|
|
max_compressed_bytes: int
|
|
max_decompressed_bytes: int
|
|
max_compression_ratio: int
|
|
max_points_per_frame: int
|
|
|
|
|
|
class FrameSummary(TypedDict):
|
|
count: int
|
|
payload_bytes: int
|
|
encoded_frame_bytes: int
|
|
other_count: int
|
|
other_payload_bytes: int
|
|
|
|
|
|
class DecodeSummary(TypedDict):
|
|
attempted: int
|
|
successes: int
|
|
errors: int
|
|
|
|
|
|
class PointCloudSummary(TypedDict):
|
|
frame_count: int
|
|
payload_bytes: int
|
|
decode_successes: int
|
|
decode_errors: int
|
|
points: PointSummary
|
|
scalers: ScalerSummary
|
|
compressed_bytes: ByteSummary
|
|
decompressed_bytes: ByteSummary
|
|
|
|
|
|
class PoseSummary(TypedDict):
|
|
frame_count: int
|
|
payload_bytes: int
|
|
decode_successes: int
|
|
decode_errors: int
|
|
first_to_last_displacement_meters: float | None
|
|
|
|
|
|
class StreamSummary(TypedDict):
|
|
schema_version: int
|
|
created_at_utc: str
|
|
sensitivity: str
|
|
source: SourceSummary
|
|
limits: LimitSummary
|
|
frames: FrameSummary
|
|
decoding: DecodeSummary
|
|
point_cloud: PointCloudSummary
|
|
pose: PoseSummary
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _IntRange:
|
|
minimum: int | None = None
|
|
maximum: int | None = None
|
|
|
|
def add(self, value: int) -> None:
|
|
if self.minimum is None or value < self.minimum:
|
|
self.minimum = value
|
|
if self.maximum is None or value > self.maximum:
|
|
self.maximum = value
|
|
|
|
def summary(self) -> IntRangeSummary:
|
|
return {"min": self.minimum, "max": self.maximum}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _PointCloudAccumulator:
|
|
frame_count: int = 0
|
|
payload_bytes: int = 0
|
|
decode_successes: int = 0
|
|
decode_errors: int = 0
|
|
point_total: int = 0
|
|
compressed_total: int = 0
|
|
decompressed_total: int = 0
|
|
points_per_frame: _IntRange = field(default_factory=_IntRange)
|
|
scalers: _IntRange = field(default_factory=_IntRange)
|
|
compressed_per_frame: _IntRange = field(default_factory=_IntRange)
|
|
decompressed_per_frame: _IntRange = field(default_factory=_IntRange)
|
|
first_scaler: int | None = None
|
|
scaler_constant: bool = True
|
|
|
|
def record_payload(self, payload_bytes: int) -> None:
|
|
self.frame_count += 1
|
|
self.payload_bytes += payload_bytes
|
|
|
|
def record_error(self) -> None:
|
|
self.decode_errors += 1
|
|
|
|
def record_decoded(
|
|
self,
|
|
*,
|
|
point_count: int,
|
|
scaler: int,
|
|
compressed_bytes: int,
|
|
decompressed_bytes: int,
|
|
) -> None:
|
|
self.decode_successes += 1
|
|
self.point_total += point_count
|
|
self.compressed_total += compressed_bytes
|
|
self.decompressed_total += decompressed_bytes
|
|
self.points_per_frame.add(point_count)
|
|
self.scalers.add(scaler)
|
|
self.compressed_per_frame.add(compressed_bytes)
|
|
self.decompressed_per_frame.add(decompressed_bytes)
|
|
if self.first_scaler is None:
|
|
self.first_scaler = scaler
|
|
elif scaler != self.first_scaler:
|
|
self.scaler_constant = False
|
|
|
|
def summary(self) -> PointCloudSummary:
|
|
scaler_summary: ScalerSummary = {
|
|
"min": self.scalers.minimum,
|
|
"max": self.scalers.maximum,
|
|
"constant": self.scaler_constant if self.decode_successes else None,
|
|
}
|
|
return {
|
|
"frame_count": self.frame_count,
|
|
"payload_bytes": self.payload_bytes,
|
|
"decode_successes": self.decode_successes,
|
|
"decode_errors": self.decode_errors,
|
|
"points": {
|
|
"total": self.point_total,
|
|
"per_frame": self.points_per_frame.summary(),
|
|
},
|
|
"scalers": scaler_summary,
|
|
"compressed_bytes": {
|
|
"total": self.compressed_total,
|
|
"per_frame": self.compressed_per_frame.summary(),
|
|
},
|
|
"decompressed_bytes": {
|
|
"total": self.decompressed_total,
|
|
"per_frame": self.decompressed_per_frame.summary(),
|
|
},
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _PoseAccumulator:
|
|
frame_count: int = 0
|
|
payload_bytes: int = 0
|
|
decode_successes: int = 0
|
|
decode_errors: int = 0
|
|
first_position: tuple[float, float, float] | None = None
|
|
last_position: tuple[float, float, float] | None = None
|
|
|
|
def record_payload(self, payload_bytes: int) -> None:
|
|
self.frame_count += 1
|
|
self.payload_bytes += payload_bytes
|
|
|
|
def record_error(self) -> None:
|
|
self.decode_errors += 1
|
|
|
|
def record_decoded(self, position: tuple[float, float, float]) -> None:
|
|
self.decode_successes += 1
|
|
if self.first_position is None:
|
|
self.first_position = position
|
|
self.last_position = position
|
|
|
|
def summary(self) -> PoseSummary:
|
|
displacement: float | None = None
|
|
if self.first_position is not None and self.last_position is not None:
|
|
candidate = math.dist(self.first_position, self.last_position)
|
|
if math.isfinite(candidate):
|
|
displacement = candidate
|
|
return {
|
|
"frame_count": self.frame_count,
|
|
"payload_bytes": self.payload_bytes,
|
|
"decode_successes": self.decode_successes,
|
|
"decode_errors": self.decode_errors,
|
|
"first_to_last_displacement_meters": displacement,
|
|
}
|
|
|
|
|
|
def summarize_mqtt_streams(
|
|
capture: Path,
|
|
*,
|
|
max_payload_bytes: int = DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
|
|
) -> StreamSummary:
|
|
"""Stream a raw MQTT capture into an aggregate-only, coordinate-free summary."""
|
|
if not 1 <= max_payload_bytes <= MAX_STREAM_SUMMARY_PAYLOAD_BYTES:
|
|
raise ValueError(
|
|
f"max_payload_bytes must be between 1 and {MAX_STREAM_SUMMARY_PAYLOAD_BYTES}"
|
|
)
|
|
|
|
capture_path = capture.expanduser()
|
|
before = capture_path.stat()
|
|
if not stat.S_ISREG(before.st_mode):
|
|
raise ValueError("capture must be a regular file")
|
|
|
|
capture_sha256 = _sha256_file(capture_path)
|
|
hashed = capture_path.stat()
|
|
if _file_identity(before) != _file_identity(hashed):
|
|
raise RuntimeError("capture changed while it was being hashed")
|
|
|
|
limits = DecodeLimits(max_mqtt_payload_bytes=max_payload_bytes)
|
|
point_cloud = _PointCloudAccumulator()
|
|
pose = _PoseAccumulator()
|
|
frame_count = 0
|
|
payload_bytes = 0
|
|
encoded_frame_bytes = 0
|
|
other_count = 0
|
|
other_payload_bytes = 0
|
|
|
|
for capture_frame in iter_capture_frames(
|
|
capture_path,
|
|
max_payload_bytes=max_payload_bytes,
|
|
):
|
|
frame_count += 1
|
|
frame_payload_bytes = len(capture_frame.payload)
|
|
payload_bytes += frame_payload_bytes
|
|
encoded_frame_bytes += capture_frame.raw_frame_bytes
|
|
|
|
if capture_frame.topic == LIO_PCL_TOPIC:
|
|
point_cloud.record_payload(frame_payload_bytes)
|
|
try:
|
|
decoded = decode_lio_pcl(capture_frame.payload, limits)
|
|
except StreamDecodeError:
|
|
point_cloud.record_error()
|
|
continue
|
|
point_cloud.record_decoded(
|
|
point_count=len(decoded.points),
|
|
scaler=decoded.header.scaler,
|
|
compressed_bytes=decoded.compressed_bytes,
|
|
decompressed_bytes=decoded.decompressed_bytes,
|
|
)
|
|
continue
|
|
|
|
if capture_frame.topic == LIO_POSE_TOPIC:
|
|
pose.record_payload(frame_payload_bytes)
|
|
try:
|
|
decoded_pose = decode_lio_pose(capture_frame.payload, limits)
|
|
except StreamDecodeError:
|
|
pose.record_error()
|
|
continue
|
|
pose.record_decoded(decoded_pose.position_xyz)
|
|
continue
|
|
|
|
# Unknown topic text is intentionally neither retained nor emitted: a malformed
|
|
# capture could place an identifier in that field.
|
|
other_count += 1
|
|
other_payload_bytes += frame_payload_bytes
|
|
|
|
after = capture_path.stat()
|
|
if _file_identity(hashed) != _file_identity(after):
|
|
raise RuntimeError("capture changed while it was being analyzed")
|
|
|
|
decode_successes = point_cloud.decode_successes + pose.decode_successes
|
|
decode_errors = point_cloud.decode_errors + pose.decode_errors
|
|
return {
|
|
"schema_version": 1,
|
|
"created_at_utc": utc_now_iso(),
|
|
"sensitivity": (
|
|
"sensitive derived K1 stream statistics; keep in ignored storage; "
|
|
"identifiers, keys, error text and coordinates are omitted"
|
|
),
|
|
"source": {
|
|
"bytes": hashed.st_size,
|
|
"sha256": capture_sha256,
|
|
},
|
|
"limits": {
|
|
"max_payload_bytes": limits.max_mqtt_payload_bytes,
|
|
"max_compressed_bytes": limits.max_compressed_bytes,
|
|
"max_decompressed_bytes": limits.max_decompressed_bytes,
|
|
"max_compression_ratio": limits.max_compression_ratio,
|
|
"max_points_per_frame": limits.max_points_per_frame,
|
|
},
|
|
"frames": {
|
|
"count": frame_count,
|
|
"payload_bytes": payload_bytes,
|
|
"encoded_frame_bytes": encoded_frame_bytes,
|
|
"other_count": other_count,
|
|
"other_payload_bytes": other_payload_bytes,
|
|
},
|
|
"decoding": {
|
|
"attempted": point_cloud.frame_count + pose.frame_count,
|
|
"successes": decode_successes,
|
|
"errors": decode_errors,
|
|
},
|
|
"point_cloud": point_cloud.summary(),
|
|
"pose": pose.summary(),
|
|
}
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(_HASH_CHUNK_BYTES), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _file_identity(file_stat: os.stat_result) -> tuple[int, int, int, int]:
|
|
return (
|
|
file_stat.st_dev,
|
|
file_stat.st_ino,
|
|
file_stat.st_size,
|
|
file_stat.st_mtime_ns,
|
|
)
|