feat(compute): add bounded camera job contract
This commit is contained in:
parent
75d2e5da4b
commit
648d5bced4
|
|
@ -0,0 +1,15 @@
|
||||||
|
"""Host-owned compute handoff contracts."""
|
||||||
|
|
||||||
|
from .jobs import (
|
||||||
|
COMPUTE_JOB_SCHEMA,
|
||||||
|
CameraComputeJob,
|
||||||
|
prepare_camera_compute_job,
|
||||||
|
validate_camera_compute_job,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"COMPUTE_JOB_SCHEMA",
|
||||||
|
"CameraComputeJob",
|
||||||
|
"prepare_camera_compute_job",
|
||||||
|
"validate_camera_compute_job",
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,414 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from k1link.artifacts import write_json_atomic
|
||||||
|
from k1link.sessions import SessionIntegrityError, inspect_recorded_media_epoch
|
||||||
|
|
||||||
|
COMPUTE_JOB_SCHEMA = "missioncore.compute-job/v1"
|
||||||
|
COMPUTE_RESULT_SCHEMA = "missioncore.compute-result/v1"
|
||||||
|
COMPUTE_PROFILE = "recorded-camera-perception/v1"
|
||||||
|
MAX_JOB_MANIFEST_BYTES = 32 * 1024 * 1024
|
||||||
|
MAX_JOB_FILES = 500_003
|
||||||
|
|
||||||
|
_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||||
|
_SAFE_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$")
|
||||||
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class CameraComputeJob:
|
||||||
|
job_id: str
|
||||||
|
job_root: Path
|
||||||
|
manifest_path: Path
|
||||||
|
input_sha256: str
|
||||||
|
input_byte_length: int
|
||||||
|
segment_count: int
|
||||||
|
timeline_start_seconds: float
|
||||||
|
timeline_end_seconds: float
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_camera_compute_job(
|
||||||
|
*,
|
||||||
|
session_root: Path,
|
||||||
|
source_id: str,
|
||||||
|
codec_epoch: int,
|
||||||
|
origin_epoch_ns: int,
|
||||||
|
origin_monotonic_ns: int,
|
||||||
|
output_root: Path,
|
||||||
|
) -> CameraComputeJob:
|
||||||
|
"""Publish one immutable, bounded camera epoch as a compute job.
|
||||||
|
|
||||||
|
The source evidence is read and validated but never rewritten. Publication
|
||||||
|
copies the canonical epoch into a private staging directory, revalidates the
|
||||||
|
copy, writes a path-free content-addressed manifest, then atomically renames
|
||||||
|
the complete job into place.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if _SAFE_COMPONENT.fullmatch(source_id) is None:
|
||||||
|
raise ValueError("camera source id is not a safe component")
|
||||||
|
if codec_epoch < 1:
|
||||||
|
raise ValueError("camera codec epoch must be positive")
|
||||||
|
if origin_epoch_ns < 0 or origin_monotonic_ns < 0:
|
||||||
|
raise ValueError("session timeline origin must be non-negative")
|
||||||
|
|
||||||
|
session = session_root.expanduser().resolve(strict=True)
|
||||||
|
if not session.is_dir() or _SAFE_COMPONENT.fullmatch(session.name) is None:
|
||||||
|
raise ValueError("observation session root is invalid")
|
||||||
|
source = (session / "media" / source_id).resolve(strict=True)
|
||||||
|
if not source.is_dir() or not source.is_relative_to(session):
|
||||||
|
raise SessionIntegrityError("camera source escapes its observation session")
|
||||||
|
epoch = (source / f"epoch-{codec_epoch}").resolve(strict=True)
|
||||||
|
if not epoch.is_dir() or epoch.parent != source:
|
||||||
|
raise SessionIntegrityError("camera epoch escapes its source")
|
||||||
|
|
||||||
|
inspected = inspect_recorded_media_epoch(
|
||||||
|
epoch,
|
||||||
|
expected_source_name=source_id,
|
||||||
|
origin_epoch_ns=origin_epoch_ns,
|
||||||
|
origin_monotonic_ns=origin_monotonic_ns,
|
||||||
|
)
|
||||||
|
file_records = _epoch_file_records(epoch, inspected.segments)
|
||||||
|
input_document = _input_document(
|
||||||
|
session_id=session.name,
|
||||||
|
source_id=source_id,
|
||||||
|
codec_epoch=codec_epoch,
|
||||||
|
inspected=inspected,
|
||||||
|
file_records=file_records,
|
||||||
|
)
|
||||||
|
input_sha256 = hashlib.sha256(_canonical_json(input_document)).hexdigest()
|
||||||
|
job_id = f"recorded-camera-{input_sha256[:24]}"
|
||||||
|
manifest = _job_manifest(job_id, input_sha256, input_document)
|
||||||
|
|
||||||
|
root = _prepare_private_directory(output_root)
|
||||||
|
final = root / job_id
|
||||||
|
if final.exists():
|
||||||
|
existing = validate_camera_compute_job(final)
|
||||||
|
if existing.input_sha256 != input_sha256:
|
||||||
|
raise SessionIntegrityError("compute job id collides with different input")
|
||||||
|
return existing
|
||||||
|
|
||||||
|
staging = root / f".tmp-{secrets.token_hex(16)}"
|
||||||
|
staging.mkdir(mode=0o700)
|
||||||
|
published = False
|
||||||
|
try:
|
||||||
|
destination_epoch = staging / "input" / "camera" / source_id / f"epoch-{codec_epoch}"
|
||||||
|
destination_epoch.mkdir(mode=0o700, parents=True)
|
||||||
|
(destination_epoch / "segments").mkdir(mode=0o700)
|
||||||
|
for record in file_records:
|
||||||
|
relative = PurePosixPath(str(record["path"]))
|
||||||
|
source_file = epoch / Path(*relative.parts[4:])
|
||||||
|
destination = staging / Path(*relative.parts)
|
||||||
|
_copy_regular_file(source_file, destination)
|
||||||
|
|
||||||
|
copied = inspect_recorded_media_epoch(
|
||||||
|
destination_epoch,
|
||||||
|
expected_source_name=source_id,
|
||||||
|
origin_epoch_ns=origin_epoch_ns,
|
||||||
|
origin_monotonic_ns=origin_monotonic_ns,
|
||||||
|
)
|
||||||
|
copied_records = _epoch_file_records(destination_epoch, copied.segments)
|
||||||
|
copied_input = _input_document(
|
||||||
|
session_id=session.name,
|
||||||
|
source_id=source_id,
|
||||||
|
codec_epoch=codec_epoch,
|
||||||
|
inspected=copied,
|
||||||
|
file_records=copied_records,
|
||||||
|
)
|
||||||
|
if _canonical_json(copied_input) != _canonical_json(input_document):
|
||||||
|
raise SessionIntegrityError("camera compute staging copy changed")
|
||||||
|
|
||||||
|
write_json_atomic(staging / "job.json", manifest)
|
||||||
|
_fsync_tree(staging)
|
||||||
|
os.replace(staging, final)
|
||||||
|
_fsync_directory(root)
|
||||||
|
published = True
|
||||||
|
finally:
|
||||||
|
if not published and staging.exists():
|
||||||
|
shutil.rmtree(staging)
|
||||||
|
|
||||||
|
return validate_camera_compute_job(final)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_camera_compute_job(job_root: Path) -> CameraComputeJob:
|
||||||
|
"""Validate a published compute job without trusting local paths in JSON."""
|
||||||
|
|
||||||
|
root = job_root.expanduser().resolve(strict=True)
|
||||||
|
if not root.is_dir() or _SAFE_JOB_ID.fullmatch(root.name) is None:
|
||||||
|
raise SessionIntegrityError("compute job root is invalid")
|
||||||
|
manifest_path = root / "job.json"
|
||||||
|
manifest = _read_json_object(manifest_path, root)
|
||||||
|
if (
|
||||||
|
manifest.get("schema_version") != COMPUTE_JOB_SCHEMA
|
||||||
|
or manifest.get("job_id") != root.name
|
||||||
|
or manifest.get("profile") != COMPUTE_PROFILE
|
||||||
|
or manifest.get("result_contract")
|
||||||
|
!= {
|
||||||
|
"schema_version": COMPUTE_RESULT_SCHEMA,
|
||||||
|
"timestamp_basis": "session-time-seconds",
|
||||||
|
}
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job manifest is incompatible")
|
||||||
|
input_sha256 = manifest.get("input_sha256")
|
||||||
|
input_document = manifest.get("input")
|
||||||
|
if (
|
||||||
|
not isinstance(input_sha256, str)
|
||||||
|
or _SHA256.fullmatch(input_sha256) is None
|
||||||
|
or not isinstance(input_document, dict)
|
||||||
|
or hashlib.sha256(_canonical_json(input_document)).hexdigest() != input_sha256
|
||||||
|
or root.name != f"recorded-camera-{input_sha256[:24]}"
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job input generation is inconsistent")
|
||||||
|
|
||||||
|
source_id = input_document.get("source_id")
|
||||||
|
codec_epoch = input_document.get("codec_epoch")
|
||||||
|
timeline = input_document.get("timeline")
|
||||||
|
files = input_document.get("files")
|
||||||
|
if (
|
||||||
|
not isinstance(source_id, str)
|
||||||
|
or _SAFE_COMPONENT.fullmatch(source_id) is None
|
||||||
|
or not isinstance(codec_epoch, int)
|
||||||
|
or isinstance(codec_epoch, bool)
|
||||||
|
or codec_epoch < 1
|
||||||
|
or not isinstance(timeline, dict)
|
||||||
|
or not isinstance(files, list)
|
||||||
|
or not 4 <= len(files) <= MAX_JOB_FILES
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job input descriptor is invalid")
|
||||||
|
|
||||||
|
expected_prefix = PurePosixPath("input") / "camera" / source_id / f"epoch-{codec_epoch}"
|
||||||
|
seen: set[str] = set()
|
||||||
|
total_bytes = 0
|
||||||
|
for record in files:
|
||||||
|
if not isinstance(record, dict):
|
||||||
|
raise SessionIntegrityError("compute job file descriptor is invalid")
|
||||||
|
relative = _safe_relative_path(record.get("path"), expected_prefix)
|
||||||
|
encoded = relative.as_posix()
|
||||||
|
if encoded in seen:
|
||||||
|
raise SessionIntegrityError("compute job file descriptor is duplicated")
|
||||||
|
seen.add(encoded)
|
||||||
|
expected_bytes = record.get("byte_length")
|
||||||
|
expected_sha256 = record.get("sha256")
|
||||||
|
if (
|
||||||
|
not isinstance(expected_bytes, int)
|
||||||
|
or isinstance(expected_bytes, bool)
|
||||||
|
or expected_bytes < 1
|
||||||
|
or not isinstance(expected_sha256, str)
|
||||||
|
or _SHA256.fullmatch(expected_sha256) is None
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job file identity is invalid")
|
||||||
|
payload_path = root / Path(*relative.parts)
|
||||||
|
metadata = _confined_regular_file(payload_path, root)
|
||||||
|
if metadata.st_size != expected_bytes or _sha256_file(payload_path) != expected_sha256:
|
||||||
|
raise SessionIntegrityError("compute job payload identity changed")
|
||||||
|
total_bytes += expected_bytes
|
||||||
|
|
||||||
|
required = {
|
||||||
|
(expected_prefix / "summary.json").as_posix(),
|
||||||
|
(expected_prefix / "index.jsonl").as_posix(),
|
||||||
|
(expected_prefix / "init.mp4").as_posix(),
|
||||||
|
}
|
||||||
|
segment_count = input_document.get("segment_count")
|
||||||
|
if not isinstance(segment_count, int) or isinstance(segment_count, bool) or segment_count < 1:
|
||||||
|
raise SessionIntegrityError("compute job segment count is invalid")
|
||||||
|
required.update(
|
||||||
|
(expected_prefix / "segments" / f"{sequence}.m4s").as_posix()
|
||||||
|
for sequence in range(1, segment_count + 1)
|
||||||
|
)
|
||||||
|
if seen != required or input_document.get("byte_length") != total_bytes:
|
||||||
|
raise SessionIntegrityError("compute job payload set is inconsistent")
|
||||||
|
|
||||||
|
start = timeline.get("start_seconds")
|
||||||
|
end = timeline.get("end_seconds")
|
||||||
|
if not isinstance(start, (int, float)) or not isinstance(end, (int, float)):
|
||||||
|
raise SessionIntegrityError("compute job timeline is invalid")
|
||||||
|
return CameraComputeJob(
|
||||||
|
job_id=root.name,
|
||||||
|
job_root=root,
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
input_sha256=input_sha256,
|
||||||
|
input_byte_length=total_bytes,
|
||||||
|
segment_count=segment_count,
|
||||||
|
timeline_start_seconds=float(start),
|
||||||
|
timeline_end_seconds=float(end),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _input_document(
|
||||||
|
*,
|
||||||
|
session_id: str,
|
||||||
|
source_id: str,
|
||||||
|
codec_epoch: int,
|
||||||
|
inspected: Any,
|
||||||
|
file_records: list[dict[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
summary_record = next(
|
||||||
|
record for record in file_records if record["path"].endswith("summary.json")
|
||||||
|
)
|
||||||
|
index_record = next(record for record in file_records if record["path"].endswith("index.jsonl"))
|
||||||
|
return {
|
||||||
|
"kind": "canonical-camera-epoch",
|
||||||
|
"session_id": session_id,
|
||||||
|
"source_id": source_id,
|
||||||
|
"codec_epoch": codec_epoch,
|
||||||
|
"synchronization": "host-arrival-best-effort",
|
||||||
|
"media_type": inspected.media_type,
|
||||||
|
"timeline": {
|
||||||
|
"basis": "session-time-seconds",
|
||||||
|
"start_seconds": inspected.timeline_start_seconds,
|
||||||
|
"end_seconds": inspected.timeline_end_seconds,
|
||||||
|
},
|
||||||
|
"segment_count": len(inspected.segments),
|
||||||
|
"byte_length": sum(int(record["byte_length"]) for record in file_records),
|
||||||
|
"archive_summary_sha256": summary_record["sha256"],
|
||||||
|
"archive_index_sha256": index_record["sha256"],
|
||||||
|
"files": file_records,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _job_manifest(job_id: str, input_sha256: str, input_document: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema_version": COMPUTE_JOB_SCHEMA,
|
||||||
|
"job_id": job_id,
|
||||||
|
"profile": COMPUTE_PROFILE,
|
||||||
|
"input_sha256": input_sha256,
|
||||||
|
"input": input_document,
|
||||||
|
"result_contract": {
|
||||||
|
"schema_version": COMPUTE_RESULT_SCHEMA,
|
||||||
|
"timestamp_basis": "session-time-seconds",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _epoch_file_records(epoch: Path, segments: tuple[Any, ...]) -> list[dict[str, Any]]:
|
||||||
|
prefix = PurePosixPath("input") / "camera" / epoch.parent.name / epoch.name
|
||||||
|
paths = [epoch / "summary.json", epoch / "index.jsonl", epoch / "init.mp4"]
|
||||||
|
paths.extend(segment.path for segment in segments)
|
||||||
|
records: list[dict[str, Any]] = []
|
||||||
|
for path in paths:
|
||||||
|
metadata = _confined_regular_file(path, epoch)
|
||||||
|
relative_inside_epoch = path.relative_to(epoch)
|
||||||
|
records.append(
|
||||||
|
{
|
||||||
|
"path": (prefix / relative_inside_epoch.as_posix()).as_posix(),
|
||||||
|
"byte_length": metadata.st_size,
|
||||||
|
"sha256": _sha256_file(path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_private_directory(path: Path) -> Path:
|
||||||
|
candidate = path.expanduser()
|
||||||
|
candidate.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
metadata = candidate.lstat()
|
||||||
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
||||||
|
raise ValueError("compute output root must be a real directory")
|
||||||
|
root = candidate.resolve(strict=True)
|
||||||
|
os.chmod(root, 0o700)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _copy_regular_file(source: Path, destination: Path) -> None:
|
||||||
|
confinement_root = source.parents[1] if source.parent.name == "segments" else source.parent
|
||||||
|
_confined_regular_file(source, confinement_root)
|
||||||
|
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
with source.open("rb") as reader, destination.open("xb") as writer:
|
||||||
|
shutil.copyfileobj(reader, writer, length=1024 * 1024)
|
||||||
|
writer.flush()
|
||||||
|
os.fsync(writer.fileno())
|
||||||
|
os.chmod(destination, 0o600)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_json_object(path: Path, root: Path) -> dict[str, Any]:
|
||||||
|
metadata = _confined_regular_file(path, root)
|
||||||
|
if not 0 < metadata.st_size <= MAX_JOB_MANIFEST_BYTES:
|
||||||
|
raise SessionIntegrityError("compute job manifest is outside bounds")
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise SessionIntegrityError("compute job manifest is unavailable") from exc
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise SessionIntegrityError("compute job manifest is not an object")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_relative_path(value: object, expected_prefix: PurePosixPath) -> PurePosixPath:
|
||||||
|
if not isinstance(value, str) or not value or "\\" in value:
|
||||||
|
raise SessionIntegrityError("compute job payload path is invalid")
|
||||||
|
relative = PurePosixPath(value)
|
||||||
|
if (
|
||||||
|
relative.is_absolute()
|
||||||
|
or ".." in relative.parts
|
||||||
|
or relative.parts[: len(expected_prefix.parts)] != expected_prefix.parts
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job payload path escapes its root")
|
||||||
|
return relative
|
||||||
|
|
||||||
|
|
||||||
|
def _confined_regular_file(path: Path, root: Path) -> os.stat_result:
|
||||||
|
try:
|
||||||
|
resolved_root = root.resolve(strict=True)
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
metadata = path.lstat()
|
||||||
|
except OSError as exc:
|
||||||
|
raise SessionIntegrityError("compute job payload is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not resolved.is_relative_to(resolved_root)
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("compute job payload is not a confined regular file")
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_file(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
while chunk := stream.read(1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json(value: object) -> bytes:
|
||||||
|
try:
|
||||||
|
payload = json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
allow_nan=False,
|
||||||
|
).encode("utf-8")
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise SessionIntegrityError("compute job descriptor cannot be encoded") from exc
|
||||||
|
if not 0 < len(payload) <= MAX_JOB_MANIFEST_BYTES:
|
||||||
|
raise SessionIntegrityError("compute job descriptor is outside bounds")
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _fsync_tree(root: Path) -> None:
|
||||||
|
directories = sorted(
|
||||||
|
(path for path in root.rglob("*") if path.is_dir()),
|
||||||
|
key=lambda path: len(path.parts),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
for directory in directories:
|
||||||
|
_fsync_directory(directory)
|
||||||
|
_fsync_directory(root)
|
||||||
|
|
||||||
|
|
||||||
|
def _fsync_directory(path: Path) -> None:
|
||||||
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||||
|
try:
|
||||||
|
os.fsync(descriptor)
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
|
@ -17,11 +17,13 @@ from rich.table import Table
|
||||||
|
|
||||||
from k1link import __version__
|
from k1link import __version__
|
||||||
from k1link.artifacts import write_json_atomic
|
from k1link.artifacts import write_json_atomic
|
||||||
|
from k1link.compute import prepare_camera_compute_job
|
||||||
from k1link.device_plugins.xgrids_k1.analyze import (
|
from k1link.device_plugins.xgrids_k1.analyze import (
|
||||||
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
|
DEFAULT_STREAM_SUMMARY_MAX_PAYLOAD_BYTES,
|
||||||
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
|
MAX_STREAM_SUMMARY_PAYLOAD_BYTES,
|
||||||
summarize_mqtt_streams,
|
summarize_mqtt_streams,
|
||||||
)
|
)
|
||||||
|
from k1link.device_plugins.xgrids_k1.archive import discover_legacy_viewer_sessions
|
||||||
from k1link.device_plugins.xgrids_k1.ble.gatt import dump_metadata
|
from k1link.device_plugins.xgrids_k1.ble.gatt import dump_metadata
|
||||||
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
|
from k1link.device_plugins.xgrids_k1.ble.scanner import scan
|
||||||
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
|
from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import (
|
||||||
|
|
@ -42,6 +44,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_authority import (
|
||||||
)
|
)
|
||||||
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
|
from k1link.device_plugins.xgrids_k1.usb.snapshot import snapshot as usb_snapshot
|
||||||
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
|
from k1link.macos_credentials import CredentialDialogError, prompt_wifi_credentials
|
||||||
|
from k1link.sessions import SessionIntegrityError
|
||||||
|
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
name="k1link",
|
name="k1link",
|
||||||
|
|
@ -57,11 +60,16 @@ authority_app = typer.Typer(
|
||||||
help="One-time local authority administration; never sends a K1 command.",
|
help="One-time local authority administration; never sends a K1 command.",
|
||||||
no_args_is_help=True,
|
no_args_is_help=True,
|
||||||
)
|
)
|
||||||
|
compute_app = typer.Typer(
|
||||||
|
help="Bounded, content-addressed handoffs to external compute workers.",
|
||||||
|
no_args_is_help=True,
|
||||||
|
)
|
||||||
app.add_typer(ble_app, name="ble")
|
app.add_typer(ble_app, name="ble")
|
||||||
app.add_typer(net_app, name="net")
|
app.add_typer(net_app, name="net")
|
||||||
app.add_typer(usb_app, name="usb")
|
app.add_typer(usb_app, name="usb")
|
||||||
app.add_typer(analyze_app, name="analyze")
|
app.add_typer(analyze_app, name="analyze")
|
||||||
app.add_typer(authority_app, name="authority")
|
app.add_typer(authority_app, name="authority")
|
||||||
|
app.add_typer(compute_app, name="compute")
|
||||||
|
|
||||||
|
|
||||||
class ToolStatus(TypedDict):
|
class ToolStatus(TypedDict):
|
||||||
|
|
@ -579,5 +587,71 @@ def analyze_mqtt_streams(
|
||||||
console.print(f"Saved: {out}")
|
console.print(f"Saved: {out}")
|
||||||
|
|
||||||
|
|
||||||
|
@compute_app.command("prepare-camera-job")
|
||||||
|
def prepare_camera_job(
|
||||||
|
session_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option(
|
||||||
|
"--session-root",
|
||||||
|
exists=True,
|
||||||
|
file_okay=False,
|
||||||
|
dir_okay=True,
|
||||||
|
readable=True,
|
||||||
|
help="Sealed observation session containing the canonical camera epoch.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
source_id: Annotated[
|
||||||
|
str,
|
||||||
|
typer.Option("--source", help="Canonical archived camera source id."),
|
||||||
|
],
|
||||||
|
codec_epoch: Annotated[
|
||||||
|
int,
|
||||||
|
typer.Option("--epoch", min=1, help="Physical camera codec epoch number."),
|
||||||
|
],
|
||||||
|
out_root: Annotated[
|
||||||
|
Path,
|
||||||
|
typer.Option(
|
||||||
|
"--out-root",
|
||||||
|
help="Ignored private root where the immutable job directory is published.",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
) -> None:
|
||||||
|
"""Validate and package one archived camera epoch without touching the K1."""
|
||||||
|
|
||||||
|
session = session_root.expanduser().resolve()
|
||||||
|
candidates = {
|
||||||
|
candidate.session_id: candidate
|
||||||
|
for candidate in discover_legacy_viewer_sessions(session.parent)
|
||||||
|
}
|
||||||
|
candidate = candidates.get(session.name)
|
||||||
|
if candidate is None or candidate.session_root != session:
|
||||||
|
console.print("[red]Compute job failed:[/red] session is not a sealed archive")
|
||||||
|
raise typer.Exit(code=2)
|
||||||
|
if all(media.source_id != source_id for media in candidate.media_sources):
|
||||||
|
console.print("[red]Compute job failed:[/red] camera source is not cataloged")
|
||||||
|
raise typer.Exit(code=2)
|
||||||
|
try:
|
||||||
|
job = prepare_camera_compute_job(
|
||||||
|
session_root=session,
|
||||||
|
source_id=source_id,
|
||||||
|
codec_epoch=codec_epoch,
|
||||||
|
origin_epoch_ns=candidate.timeline_origin_epoch_ns,
|
||||||
|
origin_monotonic_ns=candidate.timeline_origin_monotonic_ns,
|
||||||
|
output_root=out_root,
|
||||||
|
)
|
||||||
|
except (OSError, SessionIntegrityError, ValueError) as exc:
|
||||||
|
console.print(f"[red]Compute job failed:[/red] {type(exc).__name__}: {exc}")
|
||||||
|
raise typer.Exit(code=2) from exc
|
||||||
|
|
||||||
|
console.print(f"[green]Compute job ready:[/green] {job.job_id}")
|
||||||
|
console.print(
|
||||||
|
f"Segments: {job.segment_count}; bytes: {job.input_byte_length}; "
|
||||||
|
f"session time: {job.timeline_start_seconds:.6f}–"
|
||||||
|
f"{job.timeline_end_seconds:.6f} s"
|
||||||
|
)
|
||||||
|
console.print(f"Input SHA-256: {job.input_sha256}")
|
||||||
|
console.print(f"Saved: {job.job_root}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app()
|
app()
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ from .media import (
|
||||||
RecordedMediaFile,
|
RecordedMediaFile,
|
||||||
RecordedMediaInspector,
|
RecordedMediaInspector,
|
||||||
RecordedMediaManifest,
|
RecordedMediaManifest,
|
||||||
|
inspect_recorded_media_epoch,
|
||||||
validate_recorded_media_timeline,
|
validate_recorded_media_timeline,
|
||||||
)
|
)
|
||||||
from .models import (
|
from .models import (
|
||||||
|
|
@ -64,6 +65,7 @@ __all__ = [
|
||||||
"RecordedMediaFile",
|
"RecordedMediaFile",
|
||||||
"RecordedMediaInspector",
|
"RecordedMediaInspector",
|
||||||
"RecordedMediaManifest",
|
"RecordedMediaManifest",
|
||||||
|
"inspect_recorded_media_epoch",
|
||||||
"ReplayCommand",
|
"ReplayCommand",
|
||||||
"ReplayArtifact",
|
"ReplayArtifact",
|
||||||
"RecordingExporter",
|
"RecordingExporter",
|
||||||
|
|
|
||||||
|
|
@ -331,6 +331,40 @@ class RecordedMediaInspector:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_recorded_media_epoch(
|
||||||
|
epoch: Path,
|
||||||
|
*,
|
||||||
|
expected_source_name: str,
|
||||||
|
origin_epoch_ns: int,
|
||||||
|
origin_monotonic_ns: int,
|
||||||
|
) -> RecordedMediaEpoch:
|
||||||
|
"""Validate one canonical camera epoch for a bounded downstream handoff.
|
||||||
|
|
||||||
|
This is the single-epoch form of :class:`RecordedMediaInspector`. It keeps
|
||||||
|
compute-job packaging on the same digest and ISO-BMFF timing contract as
|
||||||
|
saved-session replay without requiring the worker to see the rest of the
|
||||||
|
observation session.
|
||||||
|
"""
|
||||||
|
|
||||||
|
match = _EPOCH_PATTERN.fullmatch(epoch.name)
|
||||||
|
if match is None:
|
||||||
|
raise SessionIntegrityError("recorded media epoch name is invalid")
|
||||||
|
try:
|
||||||
|
resolved = epoch.resolve(strict=True)
|
||||||
|
source = resolved.parent.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise SessionIntegrityError("recorded media epoch is unavailable") from exc
|
||||||
|
if source.name != expected_source_name or resolved.parent != source:
|
||||||
|
raise SessionIntegrityError("recorded media epoch source is inconsistent")
|
||||||
|
return _read_epoch(
|
||||||
|
resolved,
|
||||||
|
ordinal=int(match.group(1)),
|
||||||
|
expected_source_name=expected_source_name,
|
||||||
|
origin_epoch_ns=origin_epoch_ns,
|
||||||
|
origin_monotonic_ns=origin_monotonic_ns,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def validate_recorded_media_timeline(
|
def validate_recorded_media_timeline(
|
||||||
manifests: tuple[RecordedMediaManifest, ...],
|
manifests: tuple[RecordedMediaManifest, ...],
|
||||||
*,
|
*,
|
||||||
|
|
@ -863,19 +897,37 @@ def _read_epoch(
|
||||||
segment_count = summary.get("segment_count")
|
segment_count = summary.get("segment_count")
|
||||||
if not _non_negative_int(segment_count) or not 1 <= int(segment_count) <= MAX_MEDIA_SEGMENTS:
|
if not _non_negative_int(segment_count) or not 1 <= int(segment_count) <= MAX_MEDIA_SEGMENTS:
|
||||||
raise SessionIntegrityError("recorded media segment count is invalid")
|
raise SessionIntegrityError("recorded media segment count is invalid")
|
||||||
|
match = _EPOCH_PATTERN.fullmatch(epoch.name)
|
||||||
|
if (
|
||||||
|
match is None
|
||||||
|
or summary.get("codec_epoch") != int(match.group(1))
|
||||||
|
or summary.get("entry_count") != int(segment_count)
|
||||||
|
or summary.get("media_segment_count") != int(segment_count)
|
||||||
|
or summary.get("commit_policy") != "per-segment-fsync"
|
||||||
|
or summary.get("artifacts")
|
||||||
|
!= {
|
||||||
|
"init": "init.mp4",
|
||||||
|
"segments": "segments",
|
||||||
|
"index": "index.jsonl",
|
||||||
|
}
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("recorded media summary aggregate is inconsistent")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw_lines = (
|
raw_index = _read_confined_file(
|
||||||
_read_confined_file(
|
|
||||||
epoch / "index.jsonl",
|
epoch / "index.jsonl",
|
||||||
epoch,
|
epoch,
|
||||||
MAX_MEDIA_INDEX_BYTES,
|
MAX_MEDIA_INDEX_BYTES,
|
||||||
)
|
)
|
||||||
.decode("utf-8")
|
raw_lines = raw_index.decode("utf-8").splitlines()
|
||||||
.splitlines()
|
|
||||||
)
|
|
||||||
except UnicodeDecodeError as exc:
|
except UnicodeDecodeError as exc:
|
||||||
raise SessionIntegrityError("recorded media index is unavailable") from exc
|
raise SessionIntegrityError("recorded media index is unavailable") from exc
|
||||||
|
expected_index_sha256 = summary.get("index_sha256")
|
||||||
|
if (
|
||||||
|
not isinstance(expected_index_sha256, str)
|
||||||
|
or hashlib.sha256(raw_index).hexdigest() != expected_index_sha256
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("recorded media index digest changed")
|
||||||
if len(raw_lines) != int(segment_count):
|
if len(raw_lines) != int(segment_count):
|
||||||
raise SessionIntegrityError("recorded media index length does not match its summary")
|
raise SessionIntegrityError("recorded media index length does not match its summary")
|
||||||
entries: list[dict[str, Any]] = []
|
entries: list[dict[str, Any]] = []
|
||||||
|
|
@ -962,6 +1014,8 @@ def _read_epoch(
|
||||||
media_type = _mp4_media_type(init_payload)
|
media_type = _mp4_media_type(init_payload)
|
||||||
timing = _mp4_video_timing(init_payload, _Mp4ParseBudget())
|
timing = _mp4_video_timing(init_payload, _Mp4ParseBudget())
|
||||||
fragment_timings: list[_Mp4VideoFragmentTiming] = []
|
fragment_timings: list[_Mp4VideoFragmentTiming] = []
|
||||||
|
stream_sha256 = hashlib.sha256(init_payload)
|
||||||
|
valid_bytes = len(init_payload)
|
||||||
for segment in segments:
|
for segment in segments:
|
||||||
payload = _read_confined_file(
|
payload = _read_confined_file(
|
||||||
segment.path,
|
segment.path,
|
||||||
|
|
@ -970,6 +1024,8 @@ def _read_epoch(
|
||||||
)
|
)
|
||||||
if hashlib.sha256(payload).hexdigest() != segment.sha256:
|
if hashlib.sha256(payload).hexdigest() != segment.sha256:
|
||||||
raise SessionIntegrityError("recorded media segment digest changed")
|
raise SessionIntegrityError("recorded media segment digest changed")
|
||||||
|
stream_sha256.update(payload)
|
||||||
|
valid_bytes += len(payload)
|
||||||
fragment_timings.append(
|
fragment_timings.append(
|
||||||
_mp4_video_fragment_timing(
|
_mp4_video_fragment_timing(
|
||||||
payload,
|
payload,
|
||||||
|
|
@ -977,6 +1033,11 @@ def _read_epoch(
|
||||||
_Mp4ParseBudget(),
|
_Mp4ParseBudget(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
summary.get("valid_bytes") != valid_bytes
|
||||||
|
or summary.get("stream_sha256") != stream_sha256.hexdigest()
|
||||||
|
):
|
||||||
|
raise SessionIntegrityError("recorded media stream aggregate changed")
|
||||||
for previous, current in zip(
|
for previous, current in zip(
|
||||||
fragment_timings,
|
fragment_timings,
|
||||||
fragment_timings[1:],
|
fragment_timings[1:],
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,141 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from k1link.compute import prepare_camera_compute_job, validate_camera_compute_job
|
||||||
|
from k1link.sessions import SessionIntegrityError, inspect_recorded_media_epoch
|
||||||
|
from k1link.web.camera_archive import CameraArchiveWriter
|
||||||
|
|
||||||
|
|
||||||
|
def _recorded_h264_fixture() -> tuple[bytes, bytes]:
|
||||||
|
def box(box_type: bytes, payload: bytes = b"") -> bytes:
|
||||||
|
return (8 + len(payload)).to_bytes(4, "big") + box_type + payload
|
||||||
|
|
||||||
|
def full_box(box_type: bytes, payload: bytes = b"", *, flags: int = 0) -> bytes:
|
||||||
|
return box(box_type, bytes([0]) + flags.to_bytes(3, "big") + payload)
|
||||||
|
|
||||||
|
track_id = 1
|
||||||
|
sample_duration = 500
|
||||||
|
tkhd = full_box(
|
||||||
|
b"tkhd",
|
||||||
|
b"\x00" * 8 + track_id.to_bytes(4, "big") + b"\x00" * 4,
|
||||||
|
)
|
||||||
|
mdhd = full_box(
|
||||||
|
b"mdhd",
|
||||||
|
b"\x00" * 8 + (1_000).to_bytes(4, "big") + b"\x00" * 4,
|
||||||
|
)
|
||||||
|
hdlr = full_box(b"hdlr", b"\x00" * 4 + b"vide")
|
||||||
|
trak = box(b"trak", tkhd + box(b"mdia", mdhd + hdlr))
|
||||||
|
trex = full_box(
|
||||||
|
b"trex",
|
||||||
|
track_id.to_bytes(4, "big")
|
||||||
|
+ (1).to_bytes(4, "big")
|
||||||
|
+ sample_duration.to_bytes(4, "big")
|
||||||
|
+ b"\x00" * 8,
|
||||||
|
)
|
||||||
|
init = box(b"ftyp", b"isom") + box(
|
||||||
|
b"moov",
|
||||||
|
trak + box(b"mvex", trex) + box(b"avcC", b"\x01\x64\x00\x28"),
|
||||||
|
)
|
||||||
|
tfhd = full_box(b"tfhd", track_id.to_bytes(4, "big"), flags=0x020000)
|
||||||
|
tfdt = full_box(b"tfdt", (0).to_bytes(4, "big"))
|
||||||
|
trun = full_box(b"trun", (1).to_bytes(4, "big"))
|
||||||
|
fragment = box(b"moof", box(b"traf", tfhd + tfdt + trun)) + box(b"mdat", b"frame")
|
||||||
|
return init, fragment
|
||||||
|
|
||||||
|
|
||||||
|
def _camera_epoch(tmp_path: Path) -> tuple[Path, bytes]:
|
||||||
|
session = tmp_path / "session-1"
|
||||||
|
session.mkdir()
|
||||||
|
init, fragment = _recorded_h264_fixture()
|
||||||
|
writer = CameraArchiveWriter(session, "sensor.camera.left", 1)
|
||||||
|
writer.append("init", init, host_epoch_ns=1_000_000_000, host_monotonic_ns=2_000_000_000)
|
||||||
|
writer.append(
|
||||||
|
"media",
|
||||||
|
fragment,
|
||||||
|
host_epoch_ns=1_500_000_000,
|
||||||
|
host_monotonic_ns=2_500_000_000,
|
||||||
|
)
|
||||||
|
writer.close()
|
||||||
|
return session, fragment
|
||||||
|
|
||||||
|
|
||||||
|
def test_camera_compute_job_is_content_addressed_and_idempotent(tmp_path: Path) -> None:
|
||||||
|
session, fragment = _camera_epoch(tmp_path)
|
||||||
|
source_epoch = session / "media" / "sensor.camera.left" / "epoch-1"
|
||||||
|
source_identity = {
|
||||||
|
path.relative_to(source_epoch).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
for path in source_epoch.rglob("*")
|
||||||
|
if path.is_file()
|
||||||
|
}
|
||||||
|
|
||||||
|
job = prepare_camera_compute_job(
|
||||||
|
session_root=session,
|
||||||
|
source_id="sensor.camera.left",
|
||||||
|
codec_epoch=1,
|
||||||
|
origin_epoch_ns=1_000_000_000,
|
||||||
|
origin_monotonic_ns=2_000_000_000,
|
||||||
|
output_root=tmp_path / "jobs",
|
||||||
|
)
|
||||||
|
repeated = prepare_camera_compute_job(
|
||||||
|
session_root=session,
|
||||||
|
source_id="sensor.camera.left",
|
||||||
|
codec_epoch=1,
|
||||||
|
origin_epoch_ns=1_000_000_000,
|
||||||
|
origin_monotonic_ns=2_000_000_000,
|
||||||
|
output_root=tmp_path / "jobs",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repeated == job
|
||||||
|
assert job.job_id == f"recorded-camera-{job.input_sha256[:24]}"
|
||||||
|
assert job.segment_count == 1
|
||||||
|
assert job.timeline_start_seconds == 0.0
|
||||||
|
assert job.timeline_end_seconds == 0.5
|
||||||
|
copied = (
|
||||||
|
job.job_root / "input" / "camera" / "sensor.camera.left" / "epoch-1" / "segments" / "1.m4s"
|
||||||
|
)
|
||||||
|
assert copied.read_bytes() == fragment
|
||||||
|
assert validate_camera_compute_job(job.job_root) == job
|
||||||
|
assert source_identity == {
|
||||||
|
path.relative_to(source_epoch).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
for path in source_epoch.rglob("*")
|
||||||
|
if path.is_file()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_camera_compute_job_rejects_changed_published_payload(tmp_path: Path) -> None:
|
||||||
|
session, _ = _camera_epoch(tmp_path)
|
||||||
|
job = prepare_camera_compute_job(
|
||||||
|
session_root=session,
|
||||||
|
source_id="sensor.camera.left",
|
||||||
|
codec_epoch=1,
|
||||||
|
origin_epoch_ns=1_000_000_000,
|
||||||
|
origin_monotonic_ns=2_000_000_000,
|
||||||
|
output_root=tmp_path / "jobs",
|
||||||
|
)
|
||||||
|
segment = next(job.job_root.glob("input/camera/*/epoch-1/segments/1.m4s"))
|
||||||
|
segment.write_bytes(b"changed")
|
||||||
|
|
||||||
|
with pytest.raises(SessionIntegrityError, match="payload identity changed"):
|
||||||
|
validate_camera_compute_job(job.job_root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_epoch_inspection_checks_summary_aggregate_digests(tmp_path: Path) -> None:
|
||||||
|
session, _ = _camera_epoch(tmp_path)
|
||||||
|
epoch = session / "media" / "sensor.camera.left" / "epoch-1"
|
||||||
|
summary_path = epoch / "summary.json"
|
||||||
|
summary = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||||
|
summary["index_sha256"] = "0" * 64
|
||||||
|
summary_path.write_text(json.dumps(summary), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(SessionIntegrityError, match="index digest changed"):
|
||||||
|
inspect_recorded_media_epoch(
|
||||||
|
epoch,
|
||||||
|
expected_source_name="sensor.camera.left",
|
||||||
|
origin_epoch_ns=1_000_000_000,
|
||||||
|
origin_monotonic_ns=2_000_000_000,
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue