SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(k1): complete primary acquisition lifecycle

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-17 23:03:59 +03:00
parent 9d51080d2e
commit aa3680948f
66 changed files with 6093 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/sessions/recording.py
+66 -61
View File
@@ -23,10 +23,10 @@ from .plugin_contract import (
RecordingExporter,
)
# v6 adds a real session_time=0 row to the RRD itself. Older sidecars can
# declare a zero start while their payload begins at the first decoded sensor
# frame, so accepting them would violate the browser playback contract.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v7"
# v9 derives both real RRD boundary rows from the durable capture-clock
# envelope. v8 used the first/last MQTT message and could still exclude a
# camera fragment produced between source startup/shutdown and those packets.
CACHE_SCHEMA = "missioncore.derived-rerun-recording-cache/v9"
COMPATIBLE_CACHE_SCHEMAS = frozenset({CACHE_SCHEMA})
RERUN_RECORDING_MEDIA_TYPE = "application/vnd.rerun.rrd"
RERUN_SESSION_TIMELINE = "session_time"
@@ -84,6 +84,7 @@ class _ValidatedArtifact:
self.media_type,
*_stat_identity(self.file_stat),
self.replay_byte_length,
self.expected_sha256,
)
@@ -358,9 +359,7 @@ class SessionRecordingMaterializer:
try:
session_roots = tuple(self.recordings_root.iterdir())
except OSError as exc:
raise RecordingMaterializationError(
"recording cache could not be scavenged"
) from exc
raise RecordingMaterializationError("recording cache could not be scavenged") from exc
for session_root in session_roots:
try:
root_stat = session_root.lstat()
@@ -382,18 +381,13 @@ class SessionRecordingMaterializer:
and name.endswith(".tmp")
)
or (name.startswith(".source.") and name.endswith(".tmp"))
or (
name.startswith(".scene.rrd.cache.json.")
and name.endswith(".tmp")
)
or (name.startswith(".scene.rrd.cache.json.") and name.endswith(".tmp"))
)
if not stale:
continue
try:
child_stat = child.lstat()
if stat.S_ISDIR(child_stat.st_mode) and not stat.S_ISLNK(
child_stat.st_mode
):
if stat.S_ISDIR(child_stat.st_mode) and not stat.S_ISLNK(child_stat.st_mode):
shutil.rmtree(child)
else:
child.unlink()
@@ -497,12 +491,17 @@ class SessionRecordingMaterializer:
staged_root: Path | None = None
export_source = source.primary.path
export_artifacts = {artifact.artifact_id: artifact.path for artifact in source.artifacts}
try:
# Catalog digests bind every replay input, not only the primary
# raw stream. In particular the capture-clock artifact must be
# proven before its bounds influence an RRD export.
_validated_artifact_digests(source)
if any(
artifact.replay_byte_length != artifact.file_stat.st_size
for artifact in source.artifacts
):
staged_root, export_source = _stage_replay_prefix(
staged_root, export_source, export_artifacts = _stage_replay_prefix(
resolved_session_root,
source,
cancel_event=cancel_event,
@@ -516,6 +515,7 @@ class SessionRecordingMaterializer:
source.plugin_id,
export_source,
candidate_path,
artifacts=export_artifacts,
cancel_event=cancel_event,
activity_callback=lambda: _report_progress(
progress_callback,
@@ -527,9 +527,7 @@ class SessionRecordingMaterializer:
_report_progress(progress_callback, "finalizing", 0.9)
except PluginRecordingExportCancelled as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationCancelled(
"recording preparation was cancelled"
) from exc
raise RecordingMaterializationCancelled("recording preparation was cancelled") from exc
except PluginRecordingExportError as exc:
candidate_path.unlink(missing_ok=True)
raise RecordingMaterializationError("native capture could not be exported") from exc
@@ -548,18 +546,8 @@ class SessionRecordingMaterializer:
if source_after.identity != source.identity:
raise RecordingMaterializationError("native capture changed during RRD export")
_chmod_best_effort(candidate_path, 0o600)
source_sha256 = _sha256_prefix_stable(
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
artifact_digests = _validated_artifact_digests(source)
source_sha256 = artifact_digests[source.primary_artifact_id]
candidate = _materialized_from_export(
session_id=session_id,
source_sha256=source_sha256,
@@ -612,6 +600,7 @@ class SessionRecordingMaterializer:
source: Path,
destination: Path,
*,
artifacts: Mapping[str, Path],
cancel_event: threading.Event | None,
activity_callback: Callable[[], None],
) -> Mapping[str, object]:
@@ -622,6 +611,8 @@ class SessionRecordingMaterializer:
)
exporter = cast(RrdExporter, selected)
kwargs: dict[str, object] = {}
if _callable_accepts_keyword(exporter, "artifacts"):
kwargs["artifacts"] = dict(artifacts)
if _callable_accepts_keyword(exporter, "cancel_event"):
kwargs["cancel_event"] = cancel_event
if _callable_accepts_keyword(exporter, "activity_callback"):
@@ -705,28 +696,23 @@ class SessionRecordingMaterializer:
if document["recording_mtime_ns"] != recording_stat.st_mtime_ns:
return None
source_sha256 = _sha256_prefix_stable(
source.primary.path,
source.primary.file_stat,
source.primary.replay_byte_length,
)
if (
source.primary.expected_sha256 is not None
and source_sha256 != source.primary.expected_sha256
):
raise RecordingMaterializationError(
"native capture digest no longer matches catalog"
)
if source_sha256 != document["source_sha256"]:
return None
source_sha256: str | None = None
for cached, artifact in zip(cached_artifacts, source.artifacts, strict=True):
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog"
)
if digest != cached["sha256"]:
return None
if artifact.artifact_id == source.primary_artifact_id:
source_sha256 = digest
if source_sha256 is None or source_sha256 != document["source_sha256"]:
return None
recording_sha256 = _sha256_stable(recording_path, recording_stat)
if recording_sha256 != document["recording_sha256"]:
return None
@@ -817,9 +803,10 @@ def _validate_source(command: ReplayCommand) -> _ValidatedSource:
artifacts = getattr(command, "artifacts", None)
if not isinstance(plugin_id, str) or SESSION_ID_PATTERN.fullmatch(plugin_id) is None:
raise RecordingMaterializationError("replay command has an invalid plugin id")
if not isinstance(primary_artifact_id, str) or SESSION_ID_PATTERN.fullmatch(
primary_artifact_id
) is None:
if (
not isinstance(primary_artifact_id, str)
or SESSION_ID_PATTERN.fullmatch(primary_artifact_id) is None
):
raise RecordingMaterializationError("replay command has an invalid primary artifact")
if not isinstance(artifacts, tuple) or not artifacts:
raise RecordingMaterializationError("replay command has no source artifacts")
@@ -910,6 +897,22 @@ def _validate_source_state(source: _ValidatedSource) -> _ValidatedSource:
)
def _validated_artifact_digests(source: _ValidatedSource) -> dict[str, str]:
digests: dict[str, str] = {}
for artifact in source.artifacts:
digest = _sha256_prefix_stable(
artifact.path,
artifact.file_stat,
artifact.replay_byte_length,
)
if artifact.expected_sha256 is not None and digest != artifact.expected_sha256:
raise RecordingMaterializationError(
"recording artifact digest no longer matches catalog"
)
digests[artifact.artifact_id] = digest
return digests
def _materialized_from_export(
*,
session_id: str,
@@ -1165,11 +1168,12 @@ def _stage_replay_prefix(
*,
cancel_event: threading.Event | None = None,
activity_callback: Callable[[], None] | None = None,
) -> tuple[Path, Path]:
) -> tuple[Path, Path, dict[str, Path]]:
staged_root = session_cache_root / f".source.{uuid4().hex}.tmp"
try:
staged_root.mkdir(mode=0o700)
staged_primary: Path | None = None
staged_artifacts: dict[str, Path] = {}
for artifact in source.artifacts:
staged = staged_root / artifact.path.name
_copy_prefix_nofollow(
@@ -1180,12 +1184,13 @@ def _stage_replay_prefix(
cancel_event=cancel_event,
activity_callback=activity_callback,
)
staged_artifacts[artifact.artifact_id] = staged
if artifact.artifact_id == source.primary_artifact_id:
staged_primary = staged
if staged_primary is None:
raise RecordingMaterializationError("staged recording has no primary artifact")
_fsync_directory(staged_root)
return staged_root, staged_primary
return staged_root, staged_primary, staged_artifacts
except BaseException:
shutil.rmtree(staged_root, ignore_errors=True)
raise
@@ -1347,8 +1352,7 @@ def _callable_accepts_keyword(callback: Callable[..., object], keyword: str) ->
except (TypeError, ValueError):
return False
return keyword in parameters or any(
parameter.kind is inspect.Parameter.VAR_KEYWORD
for parameter in parameters.values()
parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in parameters.values()
)
@@ -1359,14 +1363,17 @@ def _exclusive_file_lock(path: Path) -> Any:
try:
import fcntl
except ImportError as exc: # pragma: no cover - production targets are POSIX
raise RecordingMaterializationError(
"cross-process recording lock is unavailable"
) from exc
raise RecordingMaterializationError("cross-process recording lock is unavailable") from exc
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(
os,
"O_NOFOLLOW",
0,
flags = (
os.O_RDWR
| os.O_CREAT
| getattr(os, "O_CLOEXEC", 0)
| getattr(
os,
"O_NOFOLLOW",
0,
)
)
try:
descriptor = os.open(path, flags, 0o600)
@@ -1383,9 +1390,7 @@ def _exclusive_file_lock(path: Path) -> Any:
fcntl.flock(descriptor, fcntl.LOCK_EX)
yield
except OSError as exc:
raise RecordingMaterializationError(
"cross-process recording lock failed"
) from exc
raise RecordingMaterializationError("cross-process recording lock failed") from exc
finally:
try:
fcntl.flock(descriptor, fcntl.LOCK_UN)