SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(k1): complete primary acquisition lifecycle

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-17 23:03:59 +03:00
parent 9d51080d2e
commit aa3680948f
66 changed files with 6093 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/sessions/media.py
+82 -59
View File
@@ -94,6 +94,16 @@ class _Mp4VideoTiming:
default_sample_duration: int | None
@dataclass(frozen=True, slots=True)
class _Mp4VideoFragmentTiming:
base_decode_time: int
duration_units: int
@property
def end_decode_time(self) -> int:
return self.base_decode_time + self.duration_units
@dataclass(slots=True)
class _Mp4ParseBudget:
boxes_remaining: int = MAX_MP4_BOXES
@@ -170,10 +180,13 @@ class RecordedMediaInspector:
artifact: RecordedMediaArtifact,
replay: ReplayCommand,
epoch_paths: tuple[Path, ...],
) -> tuple[
RecordedMediaManifest,
tuple[tuple[int, int, int, int], ...],
] | None:
) -> (
tuple[
RecordedMediaManifest,
tuple[tuple[int, int, int, int], ...],
]
| None
):
root = self._cache_root
if root is None:
return None
@@ -406,8 +419,7 @@ def _manifest_from_sidecar(
)
_validate_epoch_timeline(epochs)
byte_length = sum(
epoch.init_byte_length
+ sum(segment.byte_length for segment in epoch.segments)
epoch.init_byte_length + sum(segment.byte_length for segment in epoch.segments)
for epoch in epochs
)
timeline_start_seconds = min(epoch.timeline_start_seconds for epoch in epochs)
@@ -515,11 +527,7 @@ def _sidecar_identity(
expected_length: int,
) -> tuple[tuple[int, int, int, int], ...]:
value = document.get("source_identity")
if (
not isinstance(value, list)
or expected_length < 1
or len(value) != expected_length
):
if not isinstance(value, list) or expected_length < 1 or len(value) != expected_length:
raise SessionIntegrityError("recorded media preparation source identity is invalid")
identity: list[tuple[int, int, int, int]] = []
for item in value:
@@ -561,10 +569,7 @@ def _write_sidecar_atomic(root: Path, filename: str, payload: bytes) -> None:
try:
descriptor = os.open(
temporary,
os.O_WRONLY
| os.O_CREAT
| os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0),
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
dir_fd=directory_fd,
)
@@ -637,9 +642,7 @@ def _prepared_source_identity(
identity: list[tuple[int, int, int, int]] = []
for artifact in replay.artifacts:
metadata = _session_artifact_stat(artifact.path, replay.session_root)
identity.append(
(metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns)
)
identity.append((metadata.st_dev, metadata.st_ino, metadata.st_size, metadata.st_mtime_ns))
for expected_ordinal, (epoch_path, epoch) in enumerate(
zip(epoch_paths, epochs, strict=True),
start=1,
@@ -698,8 +701,7 @@ def _read_manifest(
)
_validate_epoch_timeline(epochs)
byte_length = sum(
epoch.init_byte_length
+ sum(segment.byte_length for segment in epoch.segments)
epoch.init_byte_length + sum(segment.byte_length for segment in epoch.segments)
for epoch in epochs
)
if not 0 < byte_length <= MAX_SAFE_INTEGER:
@@ -737,8 +739,7 @@ def _manifest_generation_sha256(
timeline_start_seconds = min(epoch.timeline_start_seconds for epoch in epochs)
timeline_end_seconds = max(epoch.timeline_end_seconds for epoch in epochs)
byte_length = sum(
epoch.init_byte_length
+ sum(segment.byte_length for segment in epoch.segments)
epoch.init_byte_length + sum(segment.byte_length for segment in epoch.segments)
for epoch in epochs
)
descriptor = {
@@ -798,11 +799,15 @@ def _read_epoch(
raise SessionIntegrityError("recorded media segment count is invalid")
try:
raw_lines = _read_confined_file(
epoch / "index.jsonl",
epoch,
MAX_MEDIA_INDEX_BYTES,
).decode("utf-8").splitlines()
raw_lines = (
_read_confined_file(
epoch / "index.jsonl",
epoch,
MAX_MEDIA_INDEX_BYTES,
)
.decode("utf-8")
.splitlines()
)
except UnicodeDecodeError as exc:
raise SessionIntegrityError("recorded media index is unavailable") from exc
if len(raw_lines) != int(segment_count):
@@ -886,14 +891,11 @@ def _read_epoch(
init_payload = _read_confined_file(init_path, epoch, MAX_INIT_BYTES)
init_sha256 = hashlib.sha256(init_payload).hexdigest()
expected_init_sha256 = summary.get("init_sha256")
if (
not isinstance(expected_init_sha256, str)
or expected_init_sha256 != init_sha256
):
if not isinstance(expected_init_sha256, str) or expected_init_sha256 != init_sha256:
raise SessionIntegrityError("recorded media init digest changed")
media_type = _mp4_media_type(init_payload)
timing = _mp4_video_timing(init_payload, _Mp4ParseBudget())
duration_units: list[int] = []
fragment_timings: list[_Mp4VideoFragmentTiming] = []
for segment in segments:
payload = _read_confined_file(
segment.path,
@@ -902,13 +904,21 @@ def _read_epoch(
)
if hashlib.sha256(payload).hexdigest() != segment.sha256:
raise SessionIntegrityError("recorded media segment digest changed")
duration_units.append(
_mp4_video_fragment_duration_units(
fragment_timings.append(
_mp4_video_fragment_timing(
payload,
timing,
_Mp4ParseBudget(),
)
)
for previous, current in zip(
fragment_timings,
fragment_timings[1:],
strict=False,
):
if current.base_decode_time != previous.end_decode_time:
raise SessionIntegrityError("recorded media fragment decode timeline is discontinuous")
duration_units = [timing.duration_units for timing in fragment_timings]
first_duration_seconds = _checked_fragment_duration_seconds(
duration_units[0],
timing.timescale,
@@ -919,10 +929,7 @@ def _read_epoch(
total_duration_seconds = total_duration_units / timing.timescale
timeline_start_seconds = max(0.0, timeline_points[0] - first_duration_seconds)
timeline_end_seconds = timeline_start_seconds + total_duration_seconds
if (
not math.isfinite(timeline_end_seconds)
or timeline_end_seconds < timeline_start_seconds
):
if not math.isfinite(timeline_end_seconds) or timeline_end_seconds < timeline_start_seconds:
raise SessionIntegrityError("recorded media epoch timeline is invalid")
return RecordedMediaEpoch(
ordinal=ordinal,
@@ -984,23 +991,22 @@ def _mp4_fragment_duration_seconds(init_payload: bytes, fragment_payload: bytes)
budget = _Mp4ParseBudget()
timing = _mp4_video_timing(init_payload, budget)
duration_units = _mp4_video_fragment_duration_units(
fragment_timing = _mp4_video_fragment_timing(
fragment_payload,
timing,
budget,
)
return _checked_fragment_duration_seconds(duration_units, timing.timescale)
return _checked_fragment_duration_seconds(
fragment_timing.duration_units,
timing.timescale,
)
def _checked_fragment_duration_seconds(duration_units: int, timescale: int) -> float:
if duration_units <= 0:
raise SessionIntegrityError("recorded media fragment has no positive duration")
duration = duration_units / timescale
if (
not math.isfinite(duration)
or duration <= 0
or duration > MAX_MP4_FRAGMENT_DURATION_SECONDS
):
if not math.isfinite(duration) or duration <= 0 or duration > MAX_MP4_FRAGMENT_DURATION_SECONDS:
raise SessionIntegrityError("recorded media fragment duration is outside bounds")
return duration
@@ -1069,9 +1075,7 @@ def _trak_media_timing(payload: bytes, budget: _Mp4ParseBudget) -> int | None:
raise SessionIntegrityError("recorded media track has no unique mdia box")
children = tuple(_iter_mp4_boxes(media_boxes[0], budget))
handlers = [
_parse_hdlr_type(box_payload)
for box_type, box_payload in children
if box_type == b"hdlr"
_parse_hdlr_type(box_payload) for box_type, box_payload in children if box_type == b"hdlr"
]
if len(handlers) != 1:
raise SessionIntegrityError("recorded media track handler is ambiguous")
@@ -1087,11 +1091,11 @@ def _trak_media_timing(payload: bytes, budget: _Mp4ParseBudget) -> int | None:
return timescales[0]
def _mp4_video_fragment_duration_units(
def _mp4_video_fragment_timing(
payload: bytes,
timing: _Mp4VideoTiming,
budget: _Mp4ParseBudget,
) -> int:
) -> _Mp4VideoFragmentTiming:
moof_payloads = [
box_payload
for box_type, box_payload in _iter_mp4_boxes(payload, budget)
@@ -1099,7 +1103,7 @@ def _mp4_video_fragment_duration_units(
]
if len(moof_payloads) != 1:
raise SessionIntegrityError("recorded media fragment has no unique moof box")
matching_durations: list[int] = []
matching_timings: list[_Mp4VideoFragmentTiming] = []
for box_type, traf_payload in _iter_mp4_boxes(moof_payloads[0], budget):
if box_type != b"traf":
continue
@@ -1110,18 +1114,28 @@ def _mp4_video_fragment_duration_units(
track_id, fragment_default_duration = _parse_tfhd(tfhd_payloads[0])
if track_id != timing.track_id:
continue
tfdt_payloads = [box for kind, box in boxes if kind == b"tfdt"]
if len(tfdt_payloads) != 1:
raise SessionIntegrityError("recorded media fragment tfdt is ambiguous")
base_decode_time = _parse_tfdt(tfdt_payloads[0])
trun_payloads = [box for kind, box in boxes if kind == b"trun"]
if not trun_payloads:
raise SessionIntegrityError("recorded media video fragment has no trun box")
default_duration = fragment_default_duration or timing.default_sample_duration
duration = sum(
_parse_trun_duration_units(trun, default_duration, budget)
for trun in trun_payloads
_parse_trun_duration_units(trun, default_duration, budget) for trun in trun_payloads
)
matching_durations.append(duration)
if len(matching_durations) != 1:
if base_decode_time > MAX_SAFE_INTEGER - duration:
raise SessionIntegrityError("recorded media fragment decode time is outside bounds")
matching_timings.append(
_Mp4VideoFragmentTiming(
base_decode_time=base_decode_time,
duration_units=duration,
)
)
if len(matching_timings) != 1:
raise SessionIntegrityError("recorded media fragment video track is ambiguous")
return matching_durations[0]
return matching_timings[0]
def _iter_mp4_boxes(
@@ -1195,6 +1209,17 @@ def _parse_tfhd(payload: bytes) -> tuple[int, int | None]:
return track_id, default_duration if default_duration and default_duration > 0 else None
def _parse_tfdt(payload: bytes) -> int:
version = _full_box_version(payload)
if version == 0:
return _read_u32(payload, 4, "tfdt base decode time")
if version == 1:
if len(payload) < 12:
raise SessionIntegrityError("recorded media tfdt base decode time is truncated")
return int.from_bytes(payload[4:12], "big")
raise SessionIntegrityError("recorded media tfdt version is unsupported")
def _parse_trun_duration_units(
payload: bytes,
default_duration: int | None,
@@ -1210,9 +1235,7 @@ def _parse_trun_duration_units(
cursor = _advance_box_cursor(payload, cursor, 4, "trun data offset")
if flags & 0x000004:
cursor = _advance_box_cursor(payload, cursor, 4, "trun first sample flags")
per_sample_width = sum(
4 for flag in (0x000100, 0x000200, 0x000400, 0x000800) if flags & flag
)
per_sample_width = sum(4 for flag in (0x000100, 0x000200, 0x000400, 0x000800) if flags & flag)
if per_sample_width and sample_count > (len(payload) - cursor) // per_sample_width:
raise SessionIntegrityError("recorded media trun samples are truncated")
if flags & 0x000100:
src/k1link/sessions/recording.py
+66 -61
View File
@@ -23,10 +23,10 @@ from .plugin_contract import (
RecordingExporter,
)
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
# declare a zero start while their payload begins at the first decoded sensor
# frame, so accepting them would violate the browser playback contract.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
# v9 derives both real RRD boundary rows from the durable capture-clock
# envelope. v8 used the first/last MQTT message and could still exclude a
# camera fragment produced between source startup/shutdown and those packets.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v9"
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
@@ -84,6 +84,7 @@ class _ValidatedArtifact:
self.media_type,
*_stat_identity(self.file_stat),
self.replay_byte_length,
self.expected_sha256,
)
@@ -358,9 +359,7 @@ class SessionRecordingMaterializer:
try:
session_roots = tuple(self.recordings_root.iterdir())
except OSError as exc:
raise RecordingMaterializationError(
"recording cache could not be scavenged"
) from exc
raise RecordingMaterializationError("recording cache could not be scavenged") from exc
for session_root in session_roots:
try:
root_stat = session_root.lstat()
@@ -382,18 +381,13 @@ class SessionRecordingMaterializer:
and name.endswith(".tmp")
)
or (name.startswith(".source.") and name.endswith(".tmp"))
or (
name.startswith(".scene.rrd.cache.json.")
and name.endswith(".tmp")
)
or (name.startswith(".scene.rrd.cache.json.") and name.endswith(".tmp"))
)
if not stale:
continue
try:
child_stat = child.lstat()
if stat.S_ISDIR(child_stat.st_mode) and not stat.S_ISLNK(
child_stat.st_mode
):
if stat.S_ISDIR(child_stat.st_mode) and not stat.S_ISLNK(child_stat.st_mode):
shutil.rmtree(child)
else:
child.unlink()
@@ -497,12 +491,17 @@ class SessionRecordingMaterializer:
staged_root: Path | None = None
export_source = source.primary.path
export_artifacts = {artifact.artifact_id: artifact.path for artifact in source.artifacts}
try:
# Catalog digests bind every replay input, not only the primary
# raw stream. In particular the capture-clock artifact must be
# proven before its bounds influence an RRD export.
_validated_artifact_digests(source)
if any(
artifact.replay_byte_length != artifact.file_stat.st_size
for artifact in source.artifacts
):
staged_root, export_source = _stage_replay_prefix(
staged_root, export_source, export_artifacts = _stage_replay_prefix(
resolved_session_root,
source,
cancel_event=cancel_event,
@@ -516,6 +515,7 @@ class SessionRecordingMaterializer:
source.plugin_id,
export_source,
candidate_path,
artifacts=export_artifacts,
cancel_event=cancel_event,
activity_callback=lambda: _report_progress(
progress_callback,
@@ -527,9 +527,7 @@ class SessionRecordingMaterializer:
_report_progress(progress_callback, "finalizing", 0.9)
except PluginRecordingExportCancelled as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationCancelled(
"recording preparation was cancelled"
) from exc
raise RecordingMaterializationCancelled("recording preparation was cancelled") from exc
except PluginRecordingExportError as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationError("native capture could not be exported") from exc
@@ -548,18 +546,8 @@ class SessionRecordingMaterializer:
if source_after.identity != source.identity:
raise RecordingMaterializationError("native capture changed during RRD export")
_chmod_best_effort(candidate_path, 0o600)
source_sha256 = _sha256_prefix_stable(
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
artifact_digests = _validated_artifact_digests(source)
source_sha256 = artifact_digests[source.primary_artifact_id]
candidate = _materialized_from_export(
session_id=session_id,
source_sha256=source_sha256,
@@ -612,6 +600,7 @@ class SessionRecordingMaterializer:
source: Path,
destination: Path,
*,
artifacts: Mapping[str, Path],
cancel_event: threading.Event | None,
activity_callback: Callable[[], None],
) -> Mapping[str, object]:
@@ -622,6 +611,8 @@ class SessionRecordingMaterializer:
)
exporter = cast(RrdExporter, selected)
kwargs: dict[str, object] = {}
if _callable_accepts_keyword(exporter, "artifacts"):
kwargs["artifacts"] = dict(artifacts)
if _callable_accepts_keyword(exporter, "cancel_event"):
kwargs["cancel_event"] = cancel_event
if _callable_accepts_keyword(exporter, "activity_callback"):
@@ -705,28 +696,23 @@ class SessionRecordingMaterializer:
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
return None
source_sha256 = _sha256_prefix_stable(
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
if source_sha256 != document["source_sha256"]:
return None
source_sha256: str | None = None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog"
)
if digest != cached["sha256"]:
return None
if artifact.artifact_id == source.primary_artifact_id:
source_sha256 = digest
if source_sha256 is None or source_sha256 != document["source_sha256"]:
return None
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
@@ -817,9 +803,10 @@ def _validate_source(command: ReplayCommand) -> _ValidatedSource:
artifacts = getattr(command, "artifacts", None)
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
raise RecordingMaterializationError("replay command has an invalid plugin id")
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
primary_artifact_id
) is None:
if (
not isinstance(primary_artifact_id, str)
or SESSION_ID_PATTERN.fullmatch(primary_artifact_id) is None
):
raise RecordingMaterializationError("replay command has an invalid primary artifact")
if not isinstance(artifacts, tuple) or not artifacts:
raise RecordingMaterializationError("replay command has no source artifacts")
@@ -910,6 +897,22 @@ def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
)
def _validated_artifact_digests(source: _ValidatedSource) -> dict[str, str]:
digests: dict[str, str] = {}
for artifact in source.artifacts:
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog"
)
digests[artifact.artifact_id] = digest
return digests
def _materialized_from_export(
*,
session_id: str,
@@ -1165,11 +1168,12 @@ def _stage_replay_prefix(
*,
cancel_event: threading.Event | None = None,
activity_callback: Callable[[], None] | None = None,
) -> tuple[Path, Path]:
) -> tuple[Path, Path, dict[str, Path]]:
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
try:
staged_root.mkdir(mode=0o700)
staged_primary: Path | None = None
staged_artifacts: dict[str, Path] = {}
for artifact in source.artifacts:
staged = staged_root / artifact.path.name
_copy_prefix_nofollow(
@@ -1180,12 +1184,13 @@ def _stage_replay_prefix(
cancel_event=cancel_event,
activity_callback=activity_callback,
)
staged_artifacts[artifact.artifact_id] = staged
if artifact.artifact_id == source.primary_artifact_id:
staged_primary = staged
if staged_primary is None:
raise RecordingMaterializationError("staged recording has no primary artifact")
_fsync_directory(staged_root)
return staged_root, staged_primary
return staged_root, staged_primary, staged_artifacts
except BaseException:
shutil.rmtree(staged_root, ignore_errors=True)
raise
@@ -1347,8 +1352,7 @@ def _callable_accepts_keyword(callback: Callable[..., object], keyword: str) ->
except (TypeError, ValueError):
return False
return keyword in parameters or any(
parameter.kind is inspect.Parameter.VAR_KEYWORD
for parameter in parameters.values()
parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()
)
@@ -1359,14 +1363,17 @@ def _exclusive_file_lock(path: Path) -> Any:
try:
import fcntl
except ImportError as exc: # pragma: no cover - production targets are POSIX
raise RecordingMaterializationError(
"cross-process recording lock is unavailable"
) from exc
raise RecordingMaterializationError("cross-process recording lock is unavailable") from exc
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(
os,
"O_NOFOLLOW",
0,
flags = (
os.O_RDWR
| os.O_CREAT
| getattr(os, "O_CLOEXEC", 0)
| getattr(
os,
"O_NOFOLLOW",
0,
)
)
try:
descriptor = os.open(path, flags, 0o600)
@@ -1383,9 +1390,7 @@ def _exclusive_file_lock(path: Path) -> Any:
fcntl.flock(descriptor, fcntl.LOCK_EX)
yield
except OSError as exc:
raise RecordingMaterializationError(
"cross-process recording lock failed"
) from exc
raise RecordingMaterializationError("cross-process recording lock failed") from exc
finally:
try:
fcntl.flock(descriptor, fcntl.LOCK_UN)