DCCONSTRUCTIONS
b53d6d5a45
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
501 lines
20 KiB
Python
501 lines
20 KiB
Python
"""Validated full-epoch perception results and native recorded-video delivery."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import stat
|
|
import subprocess
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from k1link.sessions import SessionIntegrityError
|
|
|
|
from .jobs import CameraComputeJob, validate_camera_compute_job
|
|
|
|
RESULT_SCHEMA = "missioncore.recorded-perception-result/v2"
|
|
IDENTITY_SCHEMA = "missioncore.recorded-perception-identity/v2"
|
|
REPORT_SCHEMA = "missioncore.perception-run-report/v1"
|
|
FRAME_SCHEMA = "missioncore.panoptic-frame/v1"
|
|
MAX_JSON_BYTES = 64 * 1024 * 1024
|
|
MAX_FRAME_LINE_BYTES = 1024 * 1024
|
|
MAX_JOB_SCAN = 512
|
|
MAX_SAFE_INTEGER = (1 << 53) - 1
|
|
|
|
_SAFE_RESULT_ID = re.compile(r"^result-[a-f0-9]{64}$")
|
|
_SAFE_RECORDING_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PerceptionArtifact:
|
|
kind: str
|
|
path: Path
|
|
media_type: str
|
|
byte_length: int
|
|
sha256: str
|
|
schema_version: str | None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecordedPerceptionEpochResult:
|
|
result_id: str
|
|
result_root: Path
|
|
job: CameraComputeJob
|
|
created_at_utc: str
|
|
calibration_sha256: str
|
|
calibration_slot: str
|
|
artifacts: tuple[PerceptionArtifact, ...]
|
|
|
|
def artifact(self, kind: str) -> PerceptionArtifact:
|
|
matches = tuple(item for item in self.artifacts if item.kind == kind)
|
|
if len(matches) != 1:
|
|
raise SessionIntegrityError(f"perception result artifact is unavailable: {kind}")
|
|
return matches[0]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecordedPerceptionVideo:
|
|
result_id: str
|
|
session_id: str
|
|
source_id: str
|
|
public_source_id: str
|
|
label: str
|
|
path: Path
|
|
media_type: str
|
|
byte_length: int
|
|
sha256: str
|
|
timeline_start_seconds: float
|
|
timeline_end_seconds: float
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _LatestResultCacheEntry:
|
|
jobs_root_mtime_ns: int
|
|
result_parent_mtime_ns: int
|
|
file_fingerprint: tuple[tuple[str, int, int, int, int], ...]
|
|
result: RecordedPerceptionEpochResult
|
|
|
|
|
|
class RecordedPerceptionEpochStore:
|
|
"""Discover immutable full-epoch results and expose their panoptic video."""
|
|
|
|
def __init__(self, *, jobs_root: Path, results_root: Path, ffprobe_path: Path) -> None:
|
|
self.jobs_root = jobs_root.expanduser().absolute()
|
|
self.results_root = results_root.expanduser().absolute()
|
|
self.ffprobe_path = ffprobe_path.expanduser().resolve(strict=True)
|
|
self._lock = threading.Lock()
|
|
self._video_probe_cache: dict[tuple[str, int, int], tuple[float, str, int, int]] = {}
|
|
self._latest_result_cache: dict[str, _LatestResultCacheEntry] = {}
|
|
|
|
def latest(self, session_id: str) -> RecordedPerceptionEpochResult | None:
|
|
if _SAFE_RECORDING_ID.fullmatch(session_id) is None:
|
|
raise ValueError("observation session id is invalid")
|
|
with self._lock:
|
|
return self._latest_unlocked(session_id)
|
|
|
|
def video(
|
|
self,
|
|
session_id: str,
|
|
result_id: str | None = None,
|
|
) -> RecordedPerceptionVideo | None:
|
|
if _SAFE_RECORDING_ID.fullmatch(session_id) is None:
|
|
raise ValueError("observation session id is invalid")
|
|
if result_id is not None and _SAFE_RESULT_ID.fullmatch(result_id) is None:
|
|
raise ValueError("perception result id is invalid")
|
|
with self._lock:
|
|
result = (
|
|
self._latest_unlocked(session_id)
|
|
if result_id is None
|
|
else self._resolve_unlocked(session_id, result_id)
|
|
)
|
|
if result is None:
|
|
return None
|
|
artifact = result.artifact("panoptic-overlay-video")
|
|
duration, codec, width, height = self._probe_video(artifact)
|
|
expected = result.job.timeline_end_seconds - result.job.timeline_start_seconds
|
|
if codec != "h264" or width != 800 or height != 600 or abs(duration - expected) > 1:
|
|
raise SessionIntegrityError("panoptic video stream contract is invalid")
|
|
suffix = result.job.source_id.removeprefix("sensor.camera.")
|
|
return RecordedPerceptionVideo(
|
|
result_id=result.result_id,
|
|
session_id=result.job.session_id,
|
|
source_id=result.job.source_id,
|
|
public_source_id=f"recorded.perception.{suffix}",
|
|
label=f"Сегментация · камера {suffix}",
|
|
path=artifact.path,
|
|
media_type='video/mp4; codecs="avc1.640028"',
|
|
byte_length=artifact.byte_length,
|
|
sha256=artifact.sha256,
|
|
timeline_start_seconds=result.job.timeline_start_seconds,
|
|
timeline_end_seconds=result.job.timeline_end_seconds,
|
|
)
|
|
|
|
def _latest_unlocked(self, session_id: str) -> RecordedPerceptionEpochResult | None:
|
|
cached = self._latest_result_cache.get(session_id)
|
|
if cached is not None:
|
|
if self._latest_cache_is_current(cached):
|
|
return cached.result
|
|
self._latest_result_cache.pop(session_id, None)
|
|
try:
|
|
job_roots = sorted(self.jobs_root.iterdir())
|
|
except FileNotFoundError:
|
|
return None
|
|
if len(job_roots) > MAX_JOB_SCAN:
|
|
raise SessionIntegrityError("compute job catalog is outside bounds")
|
|
matches: list[RecordedPerceptionEpochResult] = []
|
|
for job_root in job_roots:
|
|
if job_root.is_symlink():
|
|
continue
|
|
try:
|
|
job = validate_camera_compute_job(job_root)
|
|
except (OSError, SessionIntegrityError):
|
|
continue
|
|
if job.session_id != session_id:
|
|
continue
|
|
parent = self.results_root / job.job_id
|
|
try:
|
|
result_roots = sorted(
|
|
path
|
|
for path in parent.iterdir()
|
|
if path.is_dir()
|
|
and not path.is_symlink()
|
|
and _SAFE_RESULT_ID.fullmatch(path.name) is not None
|
|
)
|
|
except FileNotFoundError:
|
|
continue
|
|
for result_root in result_roots:
|
|
try:
|
|
matches.append(validate_recorded_perception_epoch_result(job_root, result_root))
|
|
except (OSError, SessionIntegrityError):
|
|
continue
|
|
if not matches:
|
|
return None
|
|
latest = max(matches, key=lambda value: (value.created_at_utc, value.result_id))
|
|
self._latest_result_cache[session_id] = self._latest_cache_entry(latest)
|
|
return latest
|
|
|
|
def _latest_cache_entry(
|
|
self,
|
|
result: RecordedPerceptionEpochResult,
|
|
) -> _LatestResultCacheEntry:
|
|
result_parent = self.results_root / result.job.job_id
|
|
return _LatestResultCacheEntry(
|
|
jobs_root_mtime_ns=self.jobs_root.stat().st_mtime_ns,
|
|
result_parent_mtime_ns=result_parent.stat().st_mtime_ns,
|
|
file_fingerprint=_result_file_fingerprint(result),
|
|
result=result,
|
|
)
|
|
|
|
def _latest_cache_is_current(self, cached: _LatestResultCacheEntry) -> bool:
|
|
"""Reuse a fully validated immutable publication while it is unchanged.
|
|
|
|
New jobs and new result generations invalidate through their parent
|
|
directory mtimes. In-place changes to the manifest or any served
|
|
result artifact invalidate through the bounded file fingerprint and
|
|
force the full hash validation again.
|
|
"""
|
|
|
|
try:
|
|
result_parent = self.results_root / cached.result.job.job_id
|
|
return (
|
|
self.jobs_root.stat().st_mtime_ns == cached.jobs_root_mtime_ns
|
|
and result_parent.stat().st_mtime_ns == cached.result_parent_mtime_ns
|
|
and _result_file_fingerprint(cached.result) == cached.file_fingerprint
|
|
)
|
|
except (OSError, SessionIntegrityError):
|
|
return False
|
|
|
|
def _resolve_unlocked(
|
|
self,
|
|
session_id: str,
|
|
result_id: str,
|
|
) -> RecordedPerceptionEpochResult | None:
|
|
latest = self._latest_unlocked(session_id)
|
|
if latest is None or latest.result_id != result_id:
|
|
return None
|
|
return latest
|
|
|
|
def _probe_video(self, artifact: PerceptionArtifact) -> tuple[float, str, int, int]:
|
|
metadata = _confined_regular_file(artifact.path, artifact.path.parent)
|
|
key = (artifact.sha256, metadata.st_mtime_ns, metadata.st_size)
|
|
cached = self._video_probe_cache.get(key)
|
|
if cached is not None:
|
|
return cached
|
|
try:
|
|
completed = subprocess.run(
|
|
[
|
|
str(self.ffprobe_path),
|
|
"-v",
|
|
"error",
|
|
"-select_streams",
|
|
"v:0",
|
|
"-show_entries",
|
|
"stream=codec_name,width,height:format=duration",
|
|
"-of",
|
|
"json",
|
|
str(artifact.path),
|
|
],
|
|
capture_output=True,
|
|
check=False,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
document = json.loads(completed.stdout)
|
|
stream = document["streams"][0]
|
|
value = (
|
|
float(document["format"]["duration"]),
|
|
str(stream["codec_name"]),
|
|
int(stream["width"]),
|
|
int(stream["height"]),
|
|
)
|
|
except (OSError, subprocess.SubprocessError, ValueError, KeyError, IndexError) as exc:
|
|
raise SessionIntegrityError("panoptic video metadata is unavailable") from exc
|
|
if completed.returncode != 0 or not math.isfinite(value[0]) or value[0] <= 0:
|
|
raise SessionIntegrityError("panoptic video could not be decoded")
|
|
self._video_probe_cache = {key: value}
|
|
return value
|
|
|
|
|
|
def validate_recorded_perception_epoch_result(
|
|
job_root: Path,
|
|
result_root: Path,
|
|
) -> RecordedPerceptionEpochResult:
|
|
job = validate_camera_compute_job(job_root)
|
|
root = result_root.expanduser().resolve(strict=True)
|
|
if not root.is_dir() or _SAFE_RESULT_ID.fullmatch(root.name) is None:
|
|
raise SessionIntegrityError("full-epoch perception result root is invalid")
|
|
result = _read_json_object(root / "result.json", root)
|
|
identity = result.get("identity")
|
|
identity_sha256 = result.get("identity_sha256")
|
|
if (
|
|
result.get("schema_version") != RESULT_SCHEMA
|
|
or result.get("result_id") != root.name
|
|
or not isinstance(identity, dict)
|
|
or identity.get("schema_version") != IDENTITY_SCHEMA
|
|
or not isinstance(identity_sha256, str)
|
|
or _SHA256.fullmatch(identity_sha256) is None
|
|
or root.name != f"result-{identity_sha256}"
|
|
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
|
or result.get("job_id") != job.job_id
|
|
or result.get("input_sha256") != job.input_sha256
|
|
or result.get("session_id") != job.session_id
|
|
or result.get("source_id") != job.source_id
|
|
or result.get("codec_epoch") != job.codec_epoch
|
|
or result.get("timestamp_basis") != "session-time-seconds"
|
|
or result.get("timeline_start_seconds") != job.timeline_start_seconds
|
|
or result.get("timeline_end_seconds") != job.timeline_end_seconds
|
|
or result.get("frames_processed") != job.segment_count
|
|
):
|
|
raise SessionIntegrityError("full-epoch perception result identity is inconsistent")
|
|
if identity.get("job_id") != job.job_id or identity.get("input_sha256") != job.input_sha256:
|
|
raise SessionIntegrityError("full-epoch perception identity is not bound to its job")
|
|
calibration = identity.get("calibration")
|
|
if (
|
|
not isinstance(calibration, dict)
|
|
or not isinstance(calibration.get("content_identity_sha256"), str)
|
|
or _SHA256.fullmatch(calibration["content_identity_sha256"]) is None
|
|
or not isinstance(calibration.get("camera_slot"), str)
|
|
or not calibration["camera_slot"]
|
|
):
|
|
raise SessionIntegrityError("full-epoch perception calibration binding is invalid")
|
|
artifacts = _validate_artifacts(root, result.get("artifacts"))
|
|
report_artifact = _artifact(artifacts, "perception-run-report")
|
|
report = _read_json_object(report_artifact.path, root)
|
|
metrics = report.get("metrics")
|
|
report_input = report.get("input")
|
|
if (
|
|
report.get("schema_version") != REPORT_SCHEMA
|
|
or report.get("state") != "published"
|
|
or report.get("result_id") != root.name
|
|
or not isinstance(metrics, dict)
|
|
or metrics.get("frames_expected") != job.segment_count
|
|
or metrics.get("frames_processed") != job.segment_count
|
|
or metrics.get("frames_failed") != 0
|
|
or metrics.get("frames_skipped") != 0
|
|
or not isinstance(report_input, dict)
|
|
or report_input.get("job_id") != job.job_id
|
|
or report_input.get("input_sha256") != job.input_sha256
|
|
):
|
|
raise SessionIntegrityError("full-epoch perception run report is inconsistent")
|
|
_validate_frame_metadata(_artifact(artifacts, "panoptic-frame-metadata"), job)
|
|
created_at_utc = result.get("created_at_utc")
|
|
if not isinstance(created_at_utc, str) or not 1 <= len(created_at_utc) <= 64:
|
|
raise SessionIntegrityError("full-epoch perception creation time is invalid")
|
|
return RecordedPerceptionEpochResult(
|
|
result_id=root.name,
|
|
result_root=root,
|
|
job=job,
|
|
created_at_utc=created_at_utc,
|
|
calibration_sha256=calibration["content_identity_sha256"],
|
|
calibration_slot=calibration["camera_slot"],
|
|
artifacts=artifacts,
|
|
)
|
|
|
|
|
|
def _validate_artifacts(root: Path, raw: object) -> tuple[PerceptionArtifact, ...]:
|
|
expected = {
|
|
"panoptic-overlay-video": ("perception.mp4", "video/mp4", None),
|
|
"panoptic-mask-archive": ("masks.tar.gz", "application/gzip", None),
|
|
"panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson", FRAME_SCHEMA),
|
|
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson", None),
|
|
"perception-run-report": ("run-report.json", "application/json", REPORT_SCHEMA),
|
|
}
|
|
if not isinstance(raw, list) or len(raw) != len(expected):
|
|
raise SessionIntegrityError("full-epoch perception artifact set is invalid")
|
|
artifacts: list[PerceptionArtifact] = []
|
|
seen: set[str] = set()
|
|
for value in raw:
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("full-epoch perception artifact is invalid")
|
|
kind = value.get("kind")
|
|
if not isinstance(kind, str) or kind in seen or kind not in expected:
|
|
raise SessionIntegrityError("full-epoch perception artifact kind is invalid")
|
|
seen.add(kind)
|
|
expected_path, expected_media_type, expected_schema = expected[kind]
|
|
path = root / expected_path
|
|
metadata = _confined_regular_file(path, root)
|
|
byte_length = value.get("byte_length")
|
|
digest = value.get("sha256")
|
|
if (
|
|
value.get("path") != expected_path
|
|
or value.get("media_type") != expected_media_type
|
|
or value.get("schema_version") != expected_schema
|
|
or not isinstance(byte_length, int)
|
|
or isinstance(byte_length, bool)
|
|
or not 0 < byte_length <= MAX_SAFE_INTEGER
|
|
or metadata.st_size != byte_length
|
|
or not isinstance(digest, str)
|
|
or _SHA256.fullmatch(digest) is None
|
|
or _sha256_file(path) != digest
|
|
):
|
|
raise SessionIntegrityError("full-epoch perception artifact identity changed")
|
|
artifacts.append(
|
|
PerceptionArtifact(
|
|
kind=kind,
|
|
path=path,
|
|
media_type=expected_media_type,
|
|
byte_length=byte_length,
|
|
sha256=digest,
|
|
schema_version=expected_schema,
|
|
)
|
|
)
|
|
return tuple(artifacts)
|
|
|
|
|
|
def _artifact(artifacts: tuple[PerceptionArtifact, ...], kind: str) -> PerceptionArtifact:
|
|
matches = tuple(item for item in artifacts if item.kind == kind)
|
|
if len(matches) != 1:
|
|
raise SessionIntegrityError("full-epoch perception artifact is unavailable")
|
|
return matches[0]
|
|
|
|
|
|
def _validate_frame_metadata(artifact: PerceptionArtifact, job: CameraComputeJob) -> None:
|
|
previous = -math.inf
|
|
count = 0
|
|
try:
|
|
with artifact.path.open("rb") as stream:
|
|
for expected_index, line in enumerate(stream):
|
|
if not line or len(line) > MAX_FRAME_LINE_BYTES:
|
|
raise SessionIntegrityError("panoptic frame metadata line is outside bounds")
|
|
value = json.loads(line)
|
|
session_seconds = value.get("session_seconds") if isinstance(value, dict) else None
|
|
if (
|
|
not isinstance(value, dict)
|
|
or value.get("schema_version") != FRAME_SCHEMA
|
|
or value.get("frame_index") != expected_index
|
|
or value.get("sequence") != expected_index + 1
|
|
or not isinstance(session_seconds, (int, float))
|
|
or isinstance(session_seconds, bool)
|
|
or not math.isfinite(float(session_seconds))
|
|
or float(session_seconds) <= previous
|
|
or float(session_seconds) < job.timeline_start_seconds - 0.001
|
|
or float(session_seconds) > job.timeline_end_seconds + 0.001
|
|
or not isinstance(value.get("instances"), list)
|
|
or not isinstance(value.get("semantic_classes"), list)
|
|
):
|
|
raise SessionIntegrityError("panoptic frame metadata is inconsistent")
|
|
previous = float(session_seconds)
|
|
count += 1
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise SessionIntegrityError("panoptic frame metadata is unavailable") from exc
|
|
if count != job.segment_count:
|
|
raise SessionIntegrityError("panoptic frame metadata count changed")
|
|
|
|
|
|
def _read_json_object(path: Path, root: Path) -> dict[str, Any]:
|
|
metadata = _confined_regular_file(path, root)
|
|
if not 0 < metadata.st_size <= MAX_JSON_BYTES:
|
|
raise SessionIntegrityError("full-epoch perception JSON is outside bounds")
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise SessionIntegrityError("full-epoch perception JSON is unavailable") from exc
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("full-epoch perception JSON is not an object")
|
|
return value
|
|
|
|
|
|
def _confined_regular_file(path: Path, root: Path) -> os.stat_result:
|
|
try:
|
|
resolved_root = root.resolve(strict=True)
|
|
resolved = path.resolve(strict=True)
|
|
metadata = path.lstat()
|
|
except OSError as exc:
|
|
raise SessionIntegrityError("full-epoch perception artifact is unavailable") from exc
|
|
if (
|
|
stat.S_ISLNK(metadata.st_mode)
|
|
or not stat.S_ISREG(metadata.st_mode)
|
|
or not resolved.is_relative_to(resolved_root)
|
|
):
|
|
raise SessionIntegrityError("full-epoch perception artifact is not confined")
|
|
return metadata
|
|
|
|
|
|
def _result_file_fingerprint(
|
|
result: RecordedPerceptionEpochResult,
|
|
) -> tuple[tuple[str, int, int, int, int], ...]:
|
|
paths = (result.job.manifest_path, result.result_root / "result.json") + tuple(
|
|
artifact.path for artifact in result.artifacts
|
|
)
|
|
values: list[tuple[str, int, int, int, int]] = []
|
|
for path in paths:
|
|
root = result.job.job_root if path == result.job.manifest_path else result.result_root
|
|
metadata = _confined_regular_file(path, root)
|
|
values.append(
|
|
(
|
|
str(path),
|
|
metadata.st_dev,
|
|
metadata.st_ino,
|
|
metadata.st_size,
|
|
metadata.st_mtime_ns,
|
|
)
|
|
)
|
|
return tuple(values)
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _canonical_json(value: object) -> bytes:
|
|
try:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
except (TypeError, ValueError) as exc:
|
|
raise SessionIntegrityError("full-epoch perception identity cannot be encoded") from exc
|