285 lines
12 KiB
Python
285 lines
12 KiB
Python
"""Bounded recorded-realtime projection of a sealed replay threat ledger."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import math
|
|
import re
|
|
import statistics
|
|
from dataclasses import dataclass
|
|
from itertools import pairwise
|
|
from pathlib import Path
|
|
from threading import RLock
|
|
from typing import Final
|
|
|
|
from .geometry import RecordedGeometryStore
|
|
from .spatial_evidence import (
|
|
project_metric_obstacles_to_body,
|
|
sample_points_in_body_frame,
|
|
)
|
|
from .threat import (
|
|
DEFAULT_REPLAY_THREAT_PROFILE_PATH,
|
|
RecordedReplayBodyFrameResolver,
|
|
load_replay_threat_profile,
|
|
)
|
|
from .threat_replay import (
|
|
THREAT_REPLAY_FRAME_SCHEMA,
|
|
THREAT_REPLAY_FRAME_SCHEMA_V2,
|
|
ThreatReplayResult,
|
|
)
|
|
|
|
RECORDED_SPATIAL_TIMELINE_SCHEMA: Final = "missioncore.recorded-spatial-evidence-timeline/v1"
|
|
RECORDED_SPATIAL_CHUNK_SCHEMA: Final = "missioncore.recorded-spatial-evidence-chunk/v1"
|
|
RECORDED_SPATIAL_FRAME_SCHEMA: Final = "missioncore.recorded-spatial-evidence-frame/v1"
|
|
RECORDED_SPATIAL_POINT_LIMIT: Final = 2_000
|
|
RECORDED_SPATIAL_MAX_CHUNK_FRAMES: Final = 24
|
|
_EXPECTED_FRAME_COUNT: Final = 4_489
|
|
_SOURCE_TIME = re.compile(rb'"source_time_ns":([0-9]+)')
|
|
|
|
|
|
class RecordedThreatTimelineError(RuntimeError):
|
|
"""A bounded timeline projection escaped its sealed result or source."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecordedThreatTimelineIndex:
|
|
offsets: tuple[int, ...]
|
|
source_times_ns: tuple[int, ...]
|
|
|
|
|
|
class RecordedThreatTimeline:
|
|
"""Read bounded spatial chunks without materializing the full ledger in memory."""
|
|
|
|
def __init__(self, *, repository_root: Path, result: ThreatReplayResult) -> None:
|
|
self.repository_root = repository_root.resolve(strict=True)
|
|
self.result = result
|
|
self.frames_path = (result.result_root / "frames.jsonl").resolve(strict=True)
|
|
if self.frames_path.is_symlink() or self.frames_path.parent != result.result_root:
|
|
raise RecordedThreatTimelineError("recorded timeline frame ledger is invalid")
|
|
self.profile = load_replay_threat_profile(
|
|
self.repository_root / DEFAULT_REPLAY_THREAT_PROFILE_PATH
|
|
)
|
|
identity = result.manifest.get("identity")
|
|
if not isinstance(identity, dict):
|
|
raise RecordedThreatTimelineError("recorded timeline identity is missing")
|
|
expected_identity = {
|
|
"profile_id": self.profile.profile_id,
|
|
"profile_sha256": self.profile.profile_sha256,
|
|
"source_id": self.profile.source_id,
|
|
"source_session_id": self.profile.session_id,
|
|
"source_pack_id": self.profile.source_pack_id,
|
|
"source_pack_sha256": self.profile.source_pack_sha256,
|
|
}
|
|
if any(identity.get(key) != value for key, value in expected_identity.items()):
|
|
raise RecordedThreatTimelineError("recorded timeline escaped the threat profile")
|
|
self.store = RecordedGeometryStore.from_repository(self.repository_root)
|
|
if (
|
|
self.store.profile.source_pack_id != self.profile.source_pack_id
|
|
or self.store.profile.source_pack_sha256 != self.profile.source_pack_sha256
|
|
or self.store.profile.frame_count != _EXPECTED_FRAME_COUNT
|
|
):
|
|
raise RecordedThreatTimelineError("recorded timeline geometry identity changed")
|
|
self.body_frames = RecordedReplayBodyFrameResolver(
|
|
self.store,
|
|
profile=self.profile.body_frame,
|
|
)
|
|
self.index = _index_frame_ledger(self.frames_path)
|
|
self._lock = RLock()
|
|
|
|
def metadata(self) -> dict[str, object]:
|
|
times = self.index.source_times_ns
|
|
intervals = [(current - previous) / 1_000_000_000 for previous, current in pairwise(times)]
|
|
nominal_interval = statistics.median(intervals)
|
|
if not math.isfinite(nominal_interval) or nominal_interval <= 0:
|
|
raise RecordedThreatTimelineError("recorded timeline cadence is invalid")
|
|
return {
|
|
"schema_version": RECORDED_SPATIAL_TIMELINE_SCHEMA,
|
|
"result_id": self.result.result_id,
|
|
"recorded_source": {
|
|
"session_id": self.profile.session_id,
|
|
"source_id": self.profile.source_id,
|
|
"synchronization": "host-arrival-best-effort",
|
|
},
|
|
"frame_count": len(times),
|
|
"frame_times_ns": list(times),
|
|
"timeline_start_seconds": times[0] / 1_000_000_000,
|
|
"timeline_end_seconds": times[-1] / 1_000_000_000,
|
|
"nominal_frame_interval_seconds": nominal_interval,
|
|
"nominal_rate_hz": 1 / nominal_interval,
|
|
"max_chunk_frames": RECORDED_SPATIAL_MAX_CHUNK_FRAMES,
|
|
"point_sample_limit": RECORDED_SPATIAL_POINT_LIMIT,
|
|
"image_width": 800,
|
|
"image_height": 600,
|
|
"rig": {
|
|
"length_m": self.profile.rig.body_length_m,
|
|
"width_m": self.profile.rig.body_width_m,
|
|
"nominal_sensor_height_m": self.profile.rig.nominal_sensor_height_m,
|
|
},
|
|
"corridor": {
|
|
"forward_length_m": self.profile.corridor.forward_length_m,
|
|
"rear_margin_m": self.profile.corridor.rear_margin_m,
|
|
"half_width_m": (
|
|
self.profile.rig.body_width_m / 2 + self.profile.corridor.lateral_clearance_m
|
|
),
|
|
"prediction_horizon_seconds": (self.profile.corridor.prediction_horizon_seconds),
|
|
},
|
|
"ground_truth": False,
|
|
"authority": "replay-simulated",
|
|
"access": "read-only-bounded-recorded-replay",
|
|
}
|
|
|
|
def chunk(self, *, start_sequence: int, frame_count: int) -> dict[str, object]:
|
|
if not 0 <= start_sequence < len(self.index.offsets):
|
|
raise RecordedThreatTimelineError("recorded timeline chunk start is invalid")
|
|
if not 1 <= frame_count <= RECORDED_SPATIAL_MAX_CHUNK_FRAMES:
|
|
raise RecordedThreatTimelineError("recorded timeline chunk size is invalid")
|
|
stop = min(len(self.index.offsets), start_sequence + frame_count)
|
|
with self._lock:
|
|
frames = [self._project_frame(sequence) for sequence in range(start_sequence, stop)]
|
|
return {
|
|
"schema_version": RECORDED_SPATIAL_CHUNK_SCHEMA,
|
|
"result_id": self.result.result_id,
|
|
"start_sequence": start_sequence,
|
|
"frame_count": len(frames),
|
|
"next_sequence": stop if stop < len(self.index.offsets) else None,
|
|
"frames": frames,
|
|
"ground_truth": False,
|
|
"authority": "replay-simulated",
|
|
"access": "read-only-bounded-recorded-replay",
|
|
}
|
|
|
|
def _project_frame(self, sequence: int) -> dict[str, object]:
|
|
row = _read_frame_at(self.frames_path, self.index, sequence)
|
|
frame_id = row.get("frame_id")
|
|
if not isinstance(frame_id, str) or not frame_id:
|
|
raise RecordedThreatTimelineError("recorded timeline frame identity is invalid")
|
|
body_frame = self.body_frames.body_frame_for_frame(frame_id)
|
|
body_frame_declared = row.get("body_frame_available")
|
|
if not isinstance(body_frame_declared, bool) or body_frame_declared is not (
|
|
body_frame is not None
|
|
):
|
|
raise RecordedThreatTimelineError("recorded timeline body-frame binding changed")
|
|
source_available = row.get("source_available")
|
|
if not isinstance(source_available, bool):
|
|
raise RecordedThreatTimelineError("recorded timeline source state is invalid")
|
|
point_cloud: list[list[float]] = []
|
|
point_source_count = 0
|
|
metric_visuals: list[dict[str, object]] = []
|
|
if body_frame is not None:
|
|
points = self.store.current_points_for_frame(sequence)
|
|
if points is None or not source_available:
|
|
raise RecordedThreatTimelineError(
|
|
"recorded timeline current increment binding changed"
|
|
)
|
|
point_cloud, point_source_count = sample_points_in_body_frame(
|
|
points,
|
|
body_frame,
|
|
point_limit=RECORDED_SPATIAL_POINT_LIMIT,
|
|
)
|
|
metric_visuals = project_metric_obstacles_to_body(
|
|
_mapping_array(row.get("metric_obstacles"), "metric obstacles"),
|
|
body_frame,
|
|
occupied_voxel_size_m=self.profile.corridor.occupied_voxel_size_m,
|
|
)
|
|
assessments = _mapping_array(row.get("assessments"), "threat assessments")
|
|
camera_proposals = row.get("camera_proposals")
|
|
if not isinstance(camera_proposals, list):
|
|
raise RecordedThreatTimelineError("recorded timeline camera proposals are invalid")
|
|
return {
|
|
"schema_version": RECORDED_SPATIAL_FRAME_SCHEMA,
|
|
"sequence": sequence,
|
|
"frame_id": frame_id,
|
|
"source_time_ns": self.index.source_times_ns[sequence],
|
|
"session_seconds": self.index.source_times_ns[sequence] / 1_000_000_000,
|
|
"source_available": source_available,
|
|
"spatial_available": body_frame is not None,
|
|
"point_cloud_body_xyz_m": point_cloud,
|
|
"point_cloud_source_count": point_source_count,
|
|
"point_cloud_sample_count": len(point_cloud),
|
|
"point_cloud_layer": "current-increment",
|
|
"rolling_map_component_count": sum(
|
|
item.get("state") == "retained" for item in metric_visuals
|
|
),
|
|
"metric_obstacles": metric_visuals,
|
|
"camera_proposals": copy.deepcopy(camera_proposals),
|
|
"decision_counts": _decision_counts(assessments),
|
|
"camera_url": (
|
|
f"/api/v1/laboratory/m4-threat/results/{self.result.result_id}"
|
|
f"/timeline/frames/{sequence}/camera"
|
|
),
|
|
"ground_truth": False,
|
|
"authority": "replay-simulated",
|
|
}
|
|
|
|
|
|
def _index_frame_ledger(path: Path) -> RecordedThreatTimelineIndex:
|
|
offsets: list[int] = []
|
|
source_times: list[int] = []
|
|
with path.open("rb") as handle:
|
|
while True:
|
|
offset = handle.tell()
|
|
line = handle.readline()
|
|
if not line:
|
|
break
|
|
match = _SOURCE_TIME.search(line)
|
|
if match is None:
|
|
raise RecordedThreatTimelineError("recorded timeline source time is missing")
|
|
offsets.append(offset)
|
|
source_times.append(int(match.group(1)))
|
|
if len(offsets) != _EXPECTED_FRAME_COUNT:
|
|
raise RecordedThreatTimelineError("recorded timeline frame count changed")
|
|
if any(current <= previous for previous, current in pairwise(source_times)):
|
|
raise RecordedThreatTimelineError("recorded timeline source time is not monotonic")
|
|
return RecordedThreatTimelineIndex(tuple(offsets), tuple(source_times))
|
|
|
|
|
|
def _read_frame_at(
|
|
path: Path,
|
|
index: RecordedThreatTimelineIndex,
|
|
sequence: int,
|
|
) -> dict[str, object]:
|
|
with path.open("rb") as handle:
|
|
handle.seek(index.offsets[sequence])
|
|
line = handle.readline()
|
|
try:
|
|
row = json.loads(line)
|
|
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
|
raise RecordedThreatTimelineError("recorded timeline frame JSON is invalid") from error
|
|
if (
|
|
not isinstance(row, dict)
|
|
or row.get("schema_version")
|
|
not in {THREAT_REPLAY_FRAME_SCHEMA, THREAT_REPLAY_FRAME_SCHEMA_V2}
|
|
or row.get("sequence") != sequence
|
|
or row.get("source_time_ns") != index.source_times_ns[sequence]
|
|
):
|
|
raise RecordedThreatTimelineError("recorded timeline frame binding changed")
|
|
return row
|
|
|
|
|
|
def _mapping_array(value: object, label: str) -> list[dict[str, object]]:
|
|
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
|
raise RecordedThreatTimelineError(f"recorded timeline {label} are invalid")
|
|
return value
|
|
|
|
|
|
def _decision_counts(assessments: list[dict[str, object]]) -> dict[str, int]:
|
|
result = {"threat": 0, "not-threat": 0, "unknown": 0}
|
|
for item in assessments:
|
|
decision = item.get("decision")
|
|
if not isinstance(decision, str) or decision not in result:
|
|
raise RecordedThreatTimelineError("recorded timeline decision is invalid")
|
|
result[decision] += 1
|
|
return result
|
|
|
|
|
|
__all__ = [
|
|
"RECORDED_SPATIAL_CHUNK_SCHEMA",
|
|
"RECORDED_SPATIAL_FRAME_SCHEMA",
|
|
"RECORDED_SPATIAL_MAX_CHUNK_FRAMES",
|
|
"RECORDED_SPATIAL_POINT_LIMIT",
|
|
"RECORDED_SPATIAL_TIMELINE_SCHEMA",
|
|
"RecordedThreatTimeline",
|
|
"RecordedThreatTimelineError",
|
|
]
|