feat(observatory): ship modular AI inference labs
This commit is contained in:
@@ -0,0 +1,211 @@
|
||||
"""Reuse exact v2 inputs before decoding the source, without changing v2 identity.
|
||||
|
||||
The legacy producer hashes its own file into every pack. Keep that producer
|
||||
unchanged: this adapter only selects and verifies an existing pack, or calls
|
||||
the original builder. A cache hit is not a streaming/cold-start qualification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import lidar_replay
|
||||
from .lidar_contract import K1_LIDAR_PACK_V2_PROFILE
|
||||
from .lidar_replay import (
|
||||
DEFAULT_POSE_COVERAGE_THRESHOLD_MS,
|
||||
LIDAR_MANIFEST_NAME,
|
||||
LIDAR_REPLAY_PACK_SCHEMA,
|
||||
LidarReplayError,
|
||||
LidarReplayPackV2,
|
||||
)
|
||||
|
||||
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
|
||||
_SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_MAX_MANIFEST_BYTES = 128 * 1024
|
||||
_HASH_CHUNK_BYTES = 1024 * 1024
|
||||
type _FileStamp = tuple[int, int, int, int, int]
|
||||
|
||||
|
||||
def prepare_lidar_replay_pack_v2(
|
||||
capture_path: Path,
|
||||
output_root: Path,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
pose_coverage_threshold_ms: float = DEFAULT_POSE_COVERAGE_THRESHOLD_MS,
|
||||
) -> Path:
|
||||
"""Return the exact validated input, skipping source decode on a cache hit.
|
||||
|
||||
Only current-producer packs with exact raw/metadata/clock-origin digests may
|
||||
be reused. Their ordinary strict reader still checks artifacts, arrays,
|
||||
logical content and equivalence. Corruption fails without replacing evidence.
|
||||
No source or pack array survives this call.
|
||||
"""
|
||||
|
||||
source = capture_path.expanduser().resolve(strict=True)
|
||||
if source.name != "mqtt.raw.k1mqtt" or not source.is_file():
|
||||
raise LidarReplayError("LiDAR replay source must be mqtt.raw.k1mqtt")
|
||||
metadata = source.with_name("mqtt.metadata.jsonl")
|
||||
if not metadata.is_file():
|
||||
raise LidarReplayError("exact host timing requires mqtt.metadata.jsonl")
|
||||
if (
|
||||
not math.isfinite(pose_coverage_threshold_ms)
|
||||
or not 0 < pose_coverage_threshold_ms <= 10_000
|
||||
):
|
||||
raise LidarReplayError("pose coverage threshold is invalid")
|
||||
resolved_session = session_id or source.parents[2].name
|
||||
if _SESSION_ID.fullmatch(resolved_session) is None:
|
||||
raise LidarReplayError("LiDAR replay session id is unsafe")
|
||||
parent = output_root.expanduser().absolute()
|
||||
if parent.is_symlink():
|
||||
raise LidarReplayError("LiDAR preparation cache cannot be a symlink")
|
||||
producer = Path(lidar_replay.__file__).resolve(strict=True)
|
||||
producer_sha256, _ = _hash_regular_file(producer)
|
||||
candidates = _candidates(parent, resolved_session, producer_sha256)
|
||||
# A cold directory does not add another whole-source hash pass.
|
||||
if not candidates:
|
||||
return lidar_replay.build_lidar_replay_pack_v2(
|
||||
source,
|
||||
parent,
|
||||
session_id=resolved_session,
|
||||
pose_coverage_threshold_ms=pose_coverage_threshold_ms,
|
||||
)
|
||||
|
||||
evidence, stamps = _source_evidence(source, metadata)
|
||||
matches = [
|
||||
(root, identity)
|
||||
for root, identity in candidates
|
||||
if identity.get("source_evidence") == evidence
|
||||
]
|
||||
if len(matches) > 1:
|
||||
raise LidarReplayError("LiDAR preparation cache has ambiguous source identity")
|
||||
if not matches:
|
||||
result = lidar_replay.build_lidar_replay_pack_v2(
|
||||
source,
|
||||
parent,
|
||||
session_id=resolved_session,
|
||||
pose_coverage_threshold_ms=pose_coverage_threshold_ms,
|
||||
)
|
||||
_check_source_stamps(source, stamps)
|
||||
return result
|
||||
|
||||
root, identity = matches[0]
|
||||
pack = LidarReplayPackV2(root)
|
||||
try:
|
||||
if pack.identity != identity:
|
||||
raise LidarReplayError("LiDAR preparation cache changed during validation")
|
||||
pose_binding = pack.quality.get("pose_binding")
|
||||
if (
|
||||
not isinstance(pose_binding, dict)
|
||||
or pose_binding.get("threshold_ms") != pose_coverage_threshold_ms
|
||||
):
|
||||
# v2 did not include this report parameter in its identity. Never
|
||||
# silently return another report or overwrite the existing pack.
|
||||
raise LidarReplayError("LiDAR cached pose coverage threshold differs")
|
||||
_check_source_stamps(source, stamps)
|
||||
return root
|
||||
finally:
|
||||
pack.close()
|
||||
|
||||
|
||||
def _candidates(
|
||||
parent: Path,
|
||||
session_id: str,
|
||||
producer_sha256: str,
|
||||
) -> list[tuple[Path, dict[str, Any]]]:
|
||||
if not parent.exists():
|
||||
return []
|
||||
result: list[tuple[Path, dict[str, Any]]] = []
|
||||
for root in parent.iterdir():
|
||||
if _PACK_ID.fullmatch(root.name) is None:
|
||||
continue
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise LidarReplayError("LiDAR preparation cache entry is unsafe")
|
||||
manifest_path = root / LIDAR_MANIFEST_NAME
|
||||
if manifest_path.is_symlink():
|
||||
raise LidarReplayError("LiDAR preparation manifest cannot be a symlink")
|
||||
try:
|
||||
with manifest_path.open("rb") as stream:
|
||||
payload = stream.read(_MAX_MANIFEST_BYTES + 1)
|
||||
if len(payload) > _MAX_MANIFEST_BYTES:
|
||||
raise ValueError("manifest too large")
|
||||
manifest = json.loads(payload)
|
||||
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
||||
raise LidarReplayError("LiDAR preparation manifest is invalid") from exc
|
||||
identity = manifest.get("identity") if isinstance(manifest, dict) else None
|
||||
if not isinstance(identity, dict):
|
||||
raise LidarReplayError("LiDAR preparation identity is invalid")
|
||||
# Unrelated profiles/producers are preserved, never eagerly decoded.
|
||||
if (
|
||||
identity.get("session_id") != session_id
|
||||
or identity.get("producer_sha256") != producer_sha256
|
||||
):
|
||||
continue
|
||||
encoded = json.dumps(
|
||||
identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
|
||||
).encode()
|
||||
digest = hashlib.sha256(encoded).hexdigest()
|
||||
if (
|
||||
manifest.get("schema_version") != LIDAR_REPLAY_PACK_SCHEMA
|
||||
or identity.get("schema_version") != LIDAR_REPLAY_PACK_SCHEMA
|
||||
or manifest.get("pack_id") != root.name
|
||||
or root.name != f"lidar-replay-pack-{digest}"
|
||||
or manifest.get("identity_sha256") != digest
|
||||
or identity.get("lidar_evidence_profile") != K1_LIDAR_PACK_V2_PROFILE.to_dict()
|
||||
):
|
||||
raise LidarReplayError("LiDAR preparation identity changed")
|
||||
result.append((root.resolve(strict=True), identity))
|
||||
return result
|
||||
|
||||
|
||||
def _stamp(value: os.stat_result) -> _FileStamp:
|
||||
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns
|
||||
|
||||
|
||||
def _hash_regular_file(path: Path) -> tuple[str, _FileStamp]:
|
||||
before = path.lstat()
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise LidarReplayError("LiDAR source evidence must be a regular file")
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
if _stamp(os.fstat(stream.fileno())) != _stamp(before):
|
||||
raise LidarReplayError("LiDAR source evidence changed before hashing")
|
||||
while chunk := stream.read(_HASH_CHUNK_BYTES):
|
||||
digest.update(chunk)
|
||||
if _stamp(os.fstat(stream.fileno())) != _stamp(before):
|
||||
raise LidarReplayError("LiDAR source evidence changed during hashing")
|
||||
if _stamp(path.lstat()) != _stamp(before):
|
||||
raise LidarReplayError("LiDAR source evidence changed after hashing")
|
||||
return digest.hexdigest(), _stamp(before)
|
||||
|
||||
|
||||
def _source_evidence(
|
||||
source: Path,
|
||||
metadata: Path,
|
||||
) -> tuple[dict[str, object], dict[Path, _FileStamp]]:
|
||||
paths = {"raw": source, "metadata": metadata}
|
||||
origin = source.with_name("mqtt.timeline.origin.json")
|
||||
if origin.exists():
|
||||
paths["clock_origin"] = origin
|
||||
evidence: dict[str, object] = {}
|
||||
stamps: dict[Path, _FileStamp] = {}
|
||||
for role, path in paths.items():
|
||||
digest, stamp = _hash_regular_file(path)
|
||||
evidence[role] = {"sha256": digest, "byte_length": stamp[2]}
|
||||
stamps[path] = stamp
|
||||
return evidence, stamps
|
||||
|
||||
|
||||
def _check_source_stamps(source: Path, stamps: dict[Path, _FileStamp]) -> None:
|
||||
origin = source.with_name("mqtt.timeline.origin.json")
|
||||
if origin.exists() != (origin in stamps):
|
||||
raise LidarReplayError("LiDAR source clock origin changed during preparation")
|
||||
for path, expected in stamps.items():
|
||||
if not stat.S_ISREG(path.lstat().st_mode) or _stamp(path.lstat()) != expected:
|
||||
raise LidarReplayError("LiDAR source evidence changed during preparation")
|
||||
@@ -36,6 +36,10 @@ APPLICATION_ID: Final = "nodedc_mission_core_recorded"
|
||||
SESSION_TIMELINE: Final = "session_time"
|
||||
RENDERER_VERSION: Final = "upstream-rerun-0.36.3-encoded-optimized-v6"
|
||||
REPLAY_RENDERER_VERSION: Final = "upstream-rerun-0.36.3-canonical-replay-v1"
|
||||
CANONICAL_REPLAY_RESULT_ID: Final = re.compile(
|
||||
r"^(?:lab-v1-vegetation-shadow|m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
|
||||
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance)|ai-composition)-[a-f0-9]{64}$"
|
||||
)
|
||||
MAX_SOURCE_BYTES: Final = 768 * 1024 * 1024
|
||||
MAX_OVERLAY_BYTES: Final = 256 * 1024 * 1024
|
||||
MAX_REPLAY_BYTES: Final = 1024 * 1024 * 1024
|
||||
@@ -138,9 +142,7 @@ class CanonicalLabReplayArtifact:
|
||||
_render_lock = threading.Lock()
|
||||
_memory_cache: dict[tuple[str, str, str], CanonicalLabOverlayArtifact] = {}
|
||||
_replay_lock = threading.Lock()
|
||||
_replay_memory_cache: dict[
|
||||
tuple[str, str, str, str], CanonicalLabReplayArtifact
|
||||
] = {}
|
||||
_replay_memory_cache: dict[tuple[str, str, str, str], CanonicalLabReplayArtifact] = {}
|
||||
|
||||
|
||||
def canonical_recording_id(path: Path) -> str:
|
||||
@@ -283,9 +285,7 @@ def canonical_lab_replay(
|
||||
or not _is_sha256(base_generation_sha256)
|
||||
or _sha256(base) != base_generation_sha256
|
||||
or not _artifact_is_regular(overlay)
|
||||
or re.fullmatch(
|
||||
r"(?:lab-v1-vegetation-shadow|m49-tgs-portable-review)-[a-f0-9]{64}", result_id
|
||||
) is None
|
||||
or CANONICAL_REPLAY_RESULT_ID.fullmatch(result_id) is None
|
||||
or not recording_id
|
||||
or len(recording_id) > 128
|
||||
):
|
||||
@@ -447,12 +447,11 @@ def _verified_camera_source(root: Path, route: dict[str, Any], jobs_root: Path)
|
||||
or not isinstance(files, list)
|
||||
):
|
||||
raise CanonicalLabOverlayError("camera job source contract changed")
|
||||
epoch_prefix = PurePosixPath(
|
||||
"input/camera/sensor.camera.right"
|
||||
) / f"epoch-{source.get('codec_epoch')}"
|
||||
epoch_prefix = (
|
||||
PurePosixPath("input/camera/sensor.camera.right") / f"epoch-{source.get('codec_epoch')}"
|
||||
)
|
||||
required = [epoch_prefix / "init.mp4"] + [
|
||||
epoch_prefix / "segments" / f"{index}.m4s"
|
||||
for index in range(1, frame_count + 1)
|
||||
epoch_prefix / "segments" / f"{index}.m4s" for index in range(1, frame_count + 1)
|
||||
]
|
||||
descriptors = {
|
||||
item.get("path"): item
|
||||
@@ -609,10 +608,7 @@ def _render_overlay(
|
||||
"/perception/camera/image",
|
||||
rr.VideoFrameReference(nanoseconds=int(video_references[index])),
|
||||
)
|
||||
masks = {
|
||||
layer_id: _read_mask(archive, index)
|
||||
for layer_id, archive in archives.items()
|
||||
}
|
||||
masks = {layer_id: _read_mask(archive, index) for layer_id, archive in archives.items()}
|
||||
for layer_id, mask in masks.items():
|
||||
recording.log(
|
||||
f"/perception/camera/segmentation/{layer_id}",
|
||||
@@ -731,15 +727,17 @@ def _video_reference_timestamps(
|
||||
if (
|
||||
len(video_timestamps) < int(len(frame_times) * 0.9)
|
||||
or np.any(np.diff(video_timestamps) < 0)
|
||||
or abs(int(video_timestamps[-1]) - int(relative_frame_times[-1]))
|
||||
> 2_000_000_000
|
||||
or abs(int(video_timestamps[-1]) - int(relative_frame_times[-1])) > 2_000_000_000
|
||||
):
|
||||
raise CanonicalLabOverlayError("video proxy timeline changed")
|
||||
indices = np.searchsorted(
|
||||
video_timestamps,
|
||||
relative_frame_times,
|
||||
side="right",
|
||||
) - 1
|
||||
indices = (
|
||||
np.searchsorted(
|
||||
video_timestamps,
|
||||
relative_frame_times,
|
||||
side="right",
|
||||
)
|
||||
- 1
|
||||
)
|
||||
return video_timestamps[np.clip(indices, 0, len(video_timestamps) - 1)]
|
||||
|
||||
|
||||
@@ -770,9 +768,7 @@ def _semantic_palette(classes: list[object]) -> tuple[int, ...]:
|
||||
or not isinstance(color, list)
|
||||
or len(color) != 3
|
||||
or any(
|
||||
not isinstance(channel, int)
|
||||
or isinstance(channel, bool)
|
||||
or not 0 <= channel <= 255
|
||||
not isinstance(channel, int) or isinstance(channel, bool) or not 0 <= channel <= 255
|
||||
for channel in color
|
||||
)
|
||||
):
|
||||
@@ -820,9 +816,7 @@ def semantic_component_boxes(
|
||||
mask, class_id, minimum_pixels=minimum_pixels
|
||||
)[:12]:
|
||||
score = min(0.99, 0.5 + pixels / 20_000)
|
||||
candidates.append(
|
||||
(score, [left, top, right, bottom], f"{label} · {score:.0%}")
|
||||
)
|
||||
candidates.append((score, [left, top, right, bottom], f"{label} · {score:.0%}"))
|
||||
candidates.sort(key=lambda row: (-row[0], row[1][1], row[1][0]))
|
||||
selected = candidates[:32]
|
||||
return [row[1] for row in selected], [row[2] for row in selected]
|
||||
@@ -1001,6 +995,8 @@ def _sha256(path: Path) -> str:
|
||||
|
||||
|
||||
def _is_sha256(value: object) -> bool:
|
||||
return isinstance(value, str) and len(value) == 64 and all(
|
||||
character in "0123456789abcdef" for character in value
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == 64
|
||||
and all(character in "0123456789abcdef" for character in value)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
"""Append-only bindings from one operator composition submission to its jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.modular_composition import CompositionSpec, canonical_bytes
|
||||
|
||||
RUN_SCHEMA: Final = "missioncore.observatory-ai-composition-run/v1"
|
||||
RUN_PROJECTION_SCHEMA: Final = "missioncore.observatory-ai-composition-run-projection/v1"
|
||||
_RUN = re.compile(r"ai-composition-[a-f0-9]{64}\Z")
|
||||
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,159}\Z")
|
||||
|
||||
|
||||
class CompositionRunError(ValueError):
|
||||
"""A composition-run binding is invalid or changed after admission."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompositionRun:
|
||||
run_id: str
|
||||
source_session_id: str
|
||||
composition_sha256: str
|
||||
module_ids: tuple[str, ...]
|
||||
setup_ids: tuple[str, ...]
|
||||
job_ids: tuple[str, ...]
|
||||
created_at_utc: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _RUN.fullmatch(self.run_id) is None:
|
||||
raise CompositionRunError("invalid composition run id")
|
||||
if any(
|
||||
_ID.fullmatch(value) is None
|
||||
for value in (
|
||||
self.source_session_id,
|
||||
*self.module_ids,
|
||||
*self.setup_ids,
|
||||
*self.job_ids,
|
||||
)
|
||||
):
|
||||
raise CompositionRunError("invalid composition run identity")
|
||||
if (
|
||||
not re.fullmatch(r"[a-f0-9]{64}", self.composition_sha256)
|
||||
or not self.module_ids
|
||||
or len(self.setup_ids) != len(self.job_ids)
|
||||
or len(set(self.setup_ids)) != len(self.setup_ids)
|
||||
or len(set(self.job_ids)) != len(self.job_ids)
|
||||
):
|
||||
raise CompositionRunError("invalid composition run members")
|
||||
if not self.created_at_utc.endswith("Z"):
|
||||
raise CompositionRunError("composition run timestamp must be UTC")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": RUN_SCHEMA,
|
||||
"run_id": self.run_id,
|
||||
"source_session_id": self.source_session_id,
|
||||
"composition_sha256": self.composition_sha256,
|
||||
"module_ids": list(self.module_ids),
|
||||
"setup_ids": list(self.setup_ids),
|
||||
"job_ids": list(self.job_ids),
|
||||
"created_at_utc": self.created_at_utc,
|
||||
}
|
||||
|
||||
|
||||
class CompositionRunStore:
|
||||
def __init__(self, root: Path) -> None:
|
||||
root = root.expanduser().absolute()
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if root.is_symlink() or root.resolve() != root:
|
||||
raise CompositionRunError("composition run store must be a real directory")
|
||||
self.root = root
|
||||
|
||||
def save(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
composition: CompositionSpec,
|
||||
setup_ids: tuple[str, ...],
|
||||
job_ids: tuple[str, ...],
|
||||
idempotency_key: str,
|
||||
created_at_utc: str,
|
||||
) -> CompositionRun:
|
||||
digest = hashlib.sha256(
|
||||
canonical_bytes(
|
||||
{
|
||||
"source_session_id": source_session_id,
|
||||
"composition_sha256": composition.sha256,
|
||||
"idempotency_key": idempotency_key,
|
||||
}
|
||||
)
|
||||
).hexdigest()
|
||||
run = CompositionRun(
|
||||
run_id=f"ai-composition-{digest}",
|
||||
source_session_id=source_session_id,
|
||||
composition_sha256=composition.sha256,
|
||||
module_ids=tuple(
|
||||
node.module.module_id
|
||||
for node in composition.nodes
|
||||
if node.module.group != "preparation"
|
||||
),
|
||||
setup_ids=setup_ids,
|
||||
job_ids=job_ids,
|
||||
created_at_utc=created_at_utc,
|
||||
)
|
||||
destination = self.root / f"{run.run_id}.json"
|
||||
payload = canonical_bytes(run.as_dict())
|
||||
if destination.exists():
|
||||
if destination.is_symlink() or destination.read_bytes() != payload:
|
||||
raise CompositionRunError("composition run identity changed")
|
||||
return run
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".run-", dir=self.root)
|
||||
path = Path(temporary)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
path.chmod(0o444)
|
||||
path.rename(destination)
|
||||
finally:
|
||||
path.unlink(missing_ok=True)
|
||||
return run
|
||||
|
||||
def get(self, run_id: str) -> CompositionRun:
|
||||
if _RUN.fullmatch(run_id) is None:
|
||||
raise CompositionRunError("invalid composition run id")
|
||||
path = self.root / f"{run_id}.json"
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024:
|
||||
raise CompositionRunError("composition run is unavailable")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise CompositionRunError("composition run is unreadable") from exc
|
||||
return _decode(value)
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str | None = None,
|
||||
include_hidden: bool = True,
|
||||
) -> tuple[CompositionRun, ...]:
|
||||
values: list[CompositionRun] = []
|
||||
for path in self.root.glob("ai-composition-*.json"):
|
||||
if _RUN.fullmatch(path.stem) is None:
|
||||
continue
|
||||
run = self.get(path.stem)
|
||||
if (source_session_id is None or run.source_session_id == source_session_id) and (
|
||||
include_hidden or self.is_visible(run.run_id)
|
||||
):
|
||||
values.append(run)
|
||||
return tuple(sorted(values, key=lambda row: (row.created_at_utc, row.run_id), reverse=True))
|
||||
|
||||
def display_name(self, run_id: str) -> str | None:
|
||||
projection = self._projection(run_id)
|
||||
value = projection.get("display_name")
|
||||
return cast(str, value) if isinstance(value, str) else None
|
||||
|
||||
def is_visible(self, run_id: str) -> bool:
|
||||
return self._projection(run_id).get("visible") is not False
|
||||
|
||||
def rename_projection(self, run_id: str, display_name: str) -> str:
|
||||
self.get(run_id)
|
||||
normalized = display_name.strip()
|
||||
if not normalized or len(normalized) > 160:
|
||||
raise CompositionRunError("invalid composition run display name")
|
||||
projection = self._projection(run_id)
|
||||
self._write_projection(
|
||||
run_id, display_name=normalized, visible=projection.get("visible") is not False
|
||||
)
|
||||
return normalized
|
||||
|
||||
def delete_projection(self, run_id: str) -> None:
|
||||
self.get(run_id)
|
||||
projection = self._projection(run_id)
|
||||
self._write_projection(
|
||||
run_id,
|
||||
display_name=cast(str | None, projection.get("display_name")),
|
||||
visible=False,
|
||||
)
|
||||
|
||||
def _projection(self, run_id: str) -> dict[str, object]:
|
||||
if _RUN.fullmatch(run_id) is None:
|
||||
raise CompositionRunError("invalid composition run id")
|
||||
path = self.root / f"{run_id}.projection.json"
|
||||
if not path.exists():
|
||||
return {"display_name": None, "visible": True}
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 4 * 1024:
|
||||
raise CompositionRunError("composition run projection is unavailable")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise CompositionRunError("composition run projection is unreadable") from exc
|
||||
expected = {"schema_version", "run_id", "display_name", "visible"}
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) != expected
|
||||
or value.get("schema_version") != RUN_PROJECTION_SCHEMA
|
||||
or value.get("run_id") != run_id
|
||||
or not isinstance(value.get("visible"), bool)
|
||||
or (
|
||||
value.get("display_name") is not None
|
||||
and (
|
||||
not isinstance(value.get("display_name"), str)
|
||||
or not cast(str, value["display_name"]).strip()
|
||||
or cast(str, value["display_name"]).strip() != value["display_name"]
|
||||
or len(cast(str, value["display_name"])) > 160
|
||||
)
|
||||
)
|
||||
):
|
||||
raise CompositionRunError("invalid composition run projection")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
def _write_projection(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
display_name: str | None,
|
||||
visible: bool,
|
||||
) -> None:
|
||||
destination = self.root / f"{run_id}.projection.json"
|
||||
payload = canonical_bytes(
|
||||
{
|
||||
"schema_version": RUN_PROJECTION_SCHEMA,
|
||||
"run_id": run_id,
|
||||
"display_name": display_name,
|
||||
"visible": visible,
|
||||
}
|
||||
)
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".projection-", dir=self.root)
|
||||
path = Path(temporary)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
path.chmod(0o600)
|
||||
path.replace(destination)
|
||||
finally:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _decode(value: object) -> CompositionRun:
|
||||
keys = {
|
||||
"schema_version",
|
||||
"run_id",
|
||||
"source_session_id",
|
||||
"composition_sha256",
|
||||
"module_ids",
|
||||
"setup_ids",
|
||||
"job_ids",
|
||||
"created_at_utc",
|
||||
}
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) != keys
|
||||
or value.get("schema_version") != RUN_SCHEMA
|
||||
):
|
||||
raise CompositionRunError("invalid composition run document")
|
||||
row = cast(dict[str, object], value)
|
||||
arrays = (row["module_ids"], row["setup_ids"], row["job_ids"])
|
||||
if any(
|
||||
not isinstance(items, list) or any(not isinstance(item, str) for item in items)
|
||||
for items in arrays
|
||||
):
|
||||
raise CompositionRunError("invalid composition run member arrays")
|
||||
return CompositionRun(
|
||||
run_id=cast(str, row["run_id"]),
|
||||
source_session_id=cast(str, row["source_session_id"]),
|
||||
composition_sha256=cast(str, row["composition_sha256"]),
|
||||
module_ids=tuple(cast(list[str], row["module_ids"])),
|
||||
setup_ids=tuple(cast(list[str], row["setup_ids"])),
|
||||
job_ids=tuple(cast(list[str], row["job_ids"])),
|
||||
created_at_utc=cast(str, row["created_at_utc"]),
|
||||
)
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Strict local Observatory ontology shared by planning, publication and replay UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.modular_composition import CompositionSpec
|
||||
|
||||
ONTOLOGY_SCHEMA: Final = "missioncore.observatory-domain-ontology/v1"
|
||||
_ID = re.compile(r"[a-z][a-z0-9._-]{1,95}\Z")
|
||||
_FIELD_ID = re.compile(r"[a-z][a-z0-9_]{1,63}\Z")
|
||||
_CARDINALITIES: Final = {
|
||||
"one-to-one",
|
||||
"one-to-many",
|
||||
"many-to-one",
|
||||
"many-to-many",
|
||||
"one-to-zero-or-one",
|
||||
"many-to-zero-or-one",
|
||||
}
|
||||
|
||||
|
||||
class ObservatoryOntologyError(ValueError):
|
||||
"""The local ontology cannot answer a product query exactly."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ViewerLayer:
|
||||
layer_id: str
|
||||
pane_id: str
|
||||
label: str
|
||||
control: str
|
||||
order: int
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"layer_id": self.layer_id,
|
||||
"pane_id": self.pane_id,
|
||||
"label": self.label,
|
||||
"control": self.control,
|
||||
"order": self.order,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleProjection:
|
||||
module_id: str
|
||||
configuration_label: str
|
||||
layer_ids: tuple[str, ...]
|
||||
|
||||
|
||||
class ObservatoryDomainOntology:
|
||||
"""Versioned named-query projection; no graph service or Platform dependency."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
document: dict[str, object],
|
||||
layers: tuple[ViewerLayer, ...],
|
||||
modules: tuple[ModuleProjection, ...],
|
||||
) -> None:
|
||||
self.document = document
|
||||
self.layers = layers
|
||||
self._layers = {layer.layer_id: layer for layer in layers}
|
||||
self._modules = {module.module_id: module for module in modules}
|
||||
self._module_order = {module.module_id: order for order, module in enumerate(modules)}
|
||||
if len(self._layers) != len(layers) or len(self._modules) != len(modules):
|
||||
raise ObservatoryOntologyError("ontology identities must be unique")
|
||||
for module in modules:
|
||||
if any(layer not in self._layers for layer in module.layer_ids):
|
||||
raise ObservatoryOntologyError("module references an unknown viewer layer")
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> ObservatoryDomainOntology:
|
||||
candidate = path.expanduser().absolute()
|
||||
if (
|
||||
candidate.is_symlink()
|
||||
or not candidate.is_file()
|
||||
or candidate.stat().st_size > 1024 * 1024
|
||||
):
|
||||
raise ObservatoryOntologyError("ontology must be a bounded regular file")
|
||||
try:
|
||||
value = json.loads(candidate.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ObservatoryOntologyError("ontology is unreadable") from exc
|
||||
if not isinstance(value, dict) or value.get("schema_version") != ONTOLOGY_SCHEMA:
|
||||
raise ObservatoryOntologyError("unsupported Observatory ontology")
|
||||
required = {
|
||||
"schema_version",
|
||||
"ontology_id",
|
||||
"version",
|
||||
"owner",
|
||||
"lifecycle",
|
||||
"entities",
|
||||
"relations",
|
||||
"panes",
|
||||
"layers",
|
||||
"module_projections",
|
||||
"named_queries",
|
||||
"platform_sync",
|
||||
}
|
||||
if set(value) != required:
|
||||
raise ObservatoryOntologyError("unexpected Observatory ontology fields")
|
||||
entity_ids = _validate_entities(value["entities"])
|
||||
_validate_relations(value["relations"], entity_ids)
|
||||
pane_ids = _validate_panes(value["panes"])
|
||||
layers = tuple(_layer(item) for item in _array(value["layers"], "layers"))
|
||||
if len({layer.layer_id for layer in layers}) != len(layers):
|
||||
raise ObservatoryOntologyError("ontology layer identities must be unique")
|
||||
if any(layer.pane_id not in pane_ids for layer in layers):
|
||||
raise ObservatoryOntologyError("viewer layer references an unknown pane")
|
||||
modules = tuple(_module(item) for item in _array(value["module_projections"], "modules"))
|
||||
named = value["named_queries"]
|
||||
if named != [
|
||||
"recording.capture-context",
|
||||
"composition.configuration-label",
|
||||
"composition.member-results",
|
||||
"composition.viewer-layers",
|
||||
"composition.ready-state",
|
||||
"lab.view-profile",
|
||||
]:
|
||||
raise ObservatoryOntologyError("required named queries are absent")
|
||||
return cls(document=cast(dict[str, object], value), layers=layers, modules=modules)
|
||||
|
||||
def project_module_ids(self, module_ids: tuple[str, ...]) -> dict[str, object]:
|
||||
selected = []
|
||||
layer_ids: set[str] = set()
|
||||
for module_id in module_ids:
|
||||
projection = self._modules.get(module_id)
|
||||
if projection is None:
|
||||
raise ObservatoryOntologyError(f"module {module_id} has no ontology projection")
|
||||
selected.append(projection)
|
||||
layer_ids.update(projection.layer_ids)
|
||||
selected.sort(key=lambda row: self._module_order[row.module_id])
|
||||
layers = sorted(
|
||||
(self._layers[layer_id] for layer_id in layer_ids),
|
||||
key=lambda row: (row.pane_id, row.order),
|
||||
)
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-presentation-projection/v1",
|
||||
"modules": [
|
||||
{"module_id": row.module_id, "label": row.configuration_label} for row in selected
|
||||
],
|
||||
"configuration_label": " · ".join(row.configuration_label for row in selected),
|
||||
"viewer_layers": [row.as_dict() for row in layers],
|
||||
}
|
||||
|
||||
def project_composition(self, composition: CompositionSpec) -> dict[str, object]:
|
||||
return self.project_module_ids(
|
||||
tuple(
|
||||
node.module.module_id
|
||||
for node in composition.nodes
|
||||
if node.module.group != "preparation"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _array(value: object, label: str) -> list[object]:
|
||||
if not isinstance(value, list):
|
||||
raise ObservatoryOntologyError(f"ontology {label} must be an array")
|
||||
return value
|
||||
|
||||
|
||||
def _record(value: object, keys: set[str], label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or set(value) != keys:
|
||||
raise ObservatoryOntologyError(f"invalid ontology {label}")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _identifier(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not _ID.fullmatch(value):
|
||||
raise ObservatoryOntologyError(f"invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def _nonempty_text(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ObservatoryOntologyError(f"invalid {label}")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_entities(value: object) -> set[str]:
|
||||
identities: set[str] = set()
|
||||
for item in _array(value, "entities"):
|
||||
row = _record(item, {"id", "identity", "owner", "lifecycle"}, "entity")
|
||||
entity_id = _identifier(row["id"], "entity id")
|
||||
if entity_id in identities:
|
||||
raise ObservatoryOntologyError("ontology entity identities must be unique")
|
||||
identities.add(entity_id)
|
||||
identity_field = row["identity"]
|
||||
if not isinstance(identity_field, str) or not _FIELD_ID.fullmatch(identity_field):
|
||||
raise ObservatoryOntologyError("invalid entity identity field")
|
||||
_nonempty_text(row["owner"], "entity owner")
|
||||
_nonempty_text(row["lifecycle"], "entity lifecycle")
|
||||
if not identities:
|
||||
raise ObservatoryOntologyError("ontology entities must not be empty")
|
||||
return identities
|
||||
|
||||
|
||||
def _validate_relations(value: object, entity_ids: set[str]) -> None:
|
||||
identities: set[str] = set()
|
||||
for item in _array(value, "relations"):
|
||||
row = _record(item, {"id", "from", "to", "cardinality"}, "relation")
|
||||
relation_id = _identifier(row["id"], "relation id")
|
||||
if relation_id in identities:
|
||||
raise ObservatoryOntologyError("ontology relation identities must be unique")
|
||||
identities.add(relation_id)
|
||||
source = _identifier(row["from"], "relation source")
|
||||
target = _identifier(row["to"], "relation target")
|
||||
if source not in entity_ids or target not in entity_ids:
|
||||
raise ObservatoryOntologyError("relation references an unknown entity")
|
||||
if row["cardinality"] not in _CARDINALITIES:
|
||||
raise ObservatoryOntologyError("invalid relation cardinality")
|
||||
|
||||
|
||||
def _validate_panes(value: object) -> set[str]:
|
||||
identities: set[str] = set()
|
||||
for item in _array(value, "panes"):
|
||||
row = _record(item, {"pane_id", "label", "order"}, "pane")
|
||||
pane_id = _identifier(row["pane_id"], "pane id")
|
||||
if pane_id in identities:
|
||||
raise ObservatoryOntologyError("ontology pane identities must be unique")
|
||||
identities.add(pane_id)
|
||||
_nonempty_text(row["label"], "pane label")
|
||||
if not isinstance(row["order"], int) or isinstance(row["order"], bool):
|
||||
raise ObservatoryOntologyError("invalid pane order")
|
||||
return identities
|
||||
|
||||
|
||||
def _layer(value: object) -> ViewerLayer:
|
||||
row = _record(value, {"layer_id", "pane_id", "label", "control", "order"}, "layer")
|
||||
if (
|
||||
not isinstance(row["label"], str)
|
||||
or not row["label"]
|
||||
or row["control"] not in {"toggle", "toggle-with-settings"}
|
||||
or not isinstance(row["order"], int)
|
||||
or isinstance(row["order"], bool)
|
||||
):
|
||||
raise ObservatoryOntologyError("invalid viewer layer presentation")
|
||||
return ViewerLayer(
|
||||
_identifier(row["layer_id"], "layer id"),
|
||||
_identifier(row["pane_id"], "pane id"),
|
||||
row["label"],
|
||||
cast(str, row["control"]),
|
||||
row["order"],
|
||||
)
|
||||
|
||||
|
||||
def _module(value: object) -> ModuleProjection:
|
||||
row = _record(value, {"module_id", "configuration_label", "layers"}, "module projection")
|
||||
if not isinstance(row["configuration_label"], str) or not row["configuration_label"]:
|
||||
raise ObservatoryOntologyError("invalid module configuration label")
|
||||
layers = tuple(
|
||||
_identifier(item, "viewer layer id") for item in _array(row["layers"], "module layers")
|
||||
)
|
||||
if len(layers) != len(set(layers)):
|
||||
raise ObservatoryOntologyError("module viewer layers must be unique")
|
||||
return ModuleProjection(
|
||||
_identifier(row["module_id"], "module id"),
|
||||
row["configuration_label"],
|
||||
layers,
|
||||
)
|
||||
@@ -160,8 +160,10 @@ class InstalledLabDockerLaunch:
|
||||
"com.nodedc.definition-sha256",
|
||||
"com.nodedc.job-id",
|
||||
"com.nodedc.managed-by",
|
||||
"com.nodedc.module-id",
|
||||
"com.nodedc.package-sha256",
|
||||
"com.nodedc.product",
|
||||
"com.nodedc.role",
|
||||
"com.nodedc.stack",
|
||||
}
|
||||
if set(self.labels) != required_labels:
|
||||
@@ -170,7 +172,9 @@ class InstalledLabDockerLaunch:
|
||||
self.labels["com.nodedc.authority"] != "observation-only"
|
||||
or self.labels["com.nodedc.component"] != self.container.container_id
|
||||
or self.labels["com.nodedc.managed-by"] != "mission-core-worker"
|
||||
or self.labels["com.nodedc.module-id"] != self.container.container_id
|
||||
or self.labels["com.nodedc.product"] != "mission-core"
|
||||
or self.labels["com.nodedc.role"] != "ai-module"
|
||||
or self.labels["com.nodedc.stack"] != "observatory"
|
||||
):
|
||||
raise InstalledLabPackageRunnerError("Docker launch labels changed")
|
||||
@@ -251,14 +255,10 @@ class DockerEngineInstalledLabLauncher:
|
||||
or len(image_sha256s) != len(set(image_sha256s))
|
||||
or any(_SHA256.fullmatch(value) is None for value in image_sha256s)
|
||||
):
|
||||
raise InstalledLabPackageRunnerError(
|
||||
"installed LAB image inventory is invalid"
|
||||
)
|
||||
raise InstalledLabPackageRunnerError("installed LAB image inventory is invalid")
|
||||
if self.transport_factory is None:
|
||||
_require_local_socket(self.socket_path)
|
||||
transport: httpx.BaseTransport = httpx.HTTPTransport(
|
||||
uds=str(self.socket_path)
|
||||
)
|
||||
transport: httpx.BaseTransport = httpx.HTTPTransport(uds=str(self.socket_path))
|
||||
else:
|
||||
transport = self.transport_factory()
|
||||
with httpx.Client(
|
||||
@@ -282,7 +282,7 @@ class DockerEngineInstalledLabLauncher:
|
||||
|
||||
def _create(self, client: httpx.Client, launch: InstalledLabDockerLaunch) -> str:
|
||||
component = launch.container.container_id[:32]
|
||||
name = f"ndc-observatory-{component}-{launch.name_token}"
|
||||
name = f"ndc-mission-core-ai-module-{component}-{launch.name_token}"
|
||||
response = self._response(
|
||||
client,
|
||||
"POST",
|
||||
@@ -455,7 +455,9 @@ class InstalledLabPackageProfileRunner:
|
||||
output_root.mkdir(mode=0o700)
|
||||
steps_root = job_root / "steps"
|
||||
steps_root.mkdir(mode=0o700)
|
||||
plan_path = job_root / "run-plan.json"
|
||||
plan_root = job_root / "plan"
|
||||
plan_root.mkdir(mode=0o700)
|
||||
plan_path = plan_root / "run-plan.json"
|
||||
_write_exclusive(
|
||||
plan_path,
|
||||
canonical_json(
|
||||
@@ -530,11 +532,11 @@ class InstalledLabPackageProfileRunner:
|
||||
),
|
||||
InstalledLabDockerMount(
|
||||
_translate_work_path(
|
||||
plan_path,
|
||||
plan_path.parent,
|
||||
controller_root=self.controller_work_root,
|
||||
engine_root=self.engine_work_root,
|
||||
),
|
||||
INSTALLED_LAB_PLAN_PATH,
|
||||
str(PurePosixPath(INSTALLED_LAB_PLAN_PATH).parent),
|
||||
True,
|
||||
),
|
||||
InstalledLabDockerMount(
|
||||
@@ -586,8 +588,10 @@ class InstalledLabPackageProfileRunner:
|
||||
"com.nodedc.definition-sha256": plan.definition_sha256,
|
||||
"com.nodedc.job-id": plan.job_id,
|
||||
"com.nodedc.managed-by": "mission-core-worker",
|
||||
"com.nodedc.module-id": container.container_id,
|
||||
"com.nodedc.package-sha256": self.package.package_sha256,
|
||||
"com.nodedc.product": "mission-core",
|
||||
"com.nodedc.role": "ai-module",
|
||||
"com.nodedc.stack": "observatory",
|
||||
},
|
||||
name_token=name_token,
|
||||
|
||||
@@ -33,7 +33,7 @@ INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA: Final = (
|
||||
INSTALLED_LAB_CONTAINER_IO_SCHEMA: Final = "missioncore.observatory-installed-lab-container-io/v2"
|
||||
|
||||
INSTALLED_LAB_SOURCE_ROOT: Final = "/missioncore/input/source"
|
||||
INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/input/run-plan.json"
|
||||
INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/plan/run-plan.json"
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT: Final = "/missioncore/input/steps"
|
||||
INSTALLED_LAB_RESULT_ROOT: Final = "/missioncore/output"
|
||||
INSTALLED_LAB_WORK_ROOT: Final = "/missioncore/work"
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Durable operator display profiles for Observatory LAB results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
PROFILE_SCHEMA: Final = "missioncore.observatory-lab-view-profile/v1"
|
||||
_RESULT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,191}\Z")
|
||||
_COLOR_MODES: Final = {"intensity", "height", "distance", "rgb", "class"}
|
||||
_PALETTES: Final = {"turbo", "viridis", "plasma", "grayscale"}
|
||||
|
||||
|
||||
class LabViewProfileError(ValueError):
|
||||
"""A LAB display profile is invalid or unavailable."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LabSceneProfile:
|
||||
point_size: float
|
||||
accumulation_seconds: float
|
||||
color_mode: str
|
||||
palette: str
|
||||
show_grid: bool
|
||||
show_labels: bool
|
||||
show_camera_frustums: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if (
|
||||
isinstance(self.point_size, bool)
|
||||
or not math.isfinite(self.point_size)
|
||||
or self.point_size < 0.1
|
||||
or isinstance(self.accumulation_seconds, bool)
|
||||
or not math.isfinite(self.accumulation_seconds)
|
||||
or self.accumulation_seconds < 0
|
||||
or self.color_mode not in _COLOR_MODES
|
||||
or self.palette not in _PALETTES
|
||||
or any(
|
||||
not isinstance(value, bool)
|
||||
for value in (
|
||||
self.show_grid,
|
||||
self.show_labels,
|
||||
self.show_camera_frustums,
|
||||
)
|
||||
)
|
||||
):
|
||||
raise LabViewProfileError("invalid LAB scene profile")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"point_size": self.point_size,
|
||||
"accumulation_seconds": self.accumulation_seconds,
|
||||
"color_mode": self.color_mode,
|
||||
"palette": self.palette,
|
||||
"show_grid": self.show_grid,
|
||||
"show_labels": self.show_labels,
|
||||
"show_camera_frustums": self.show_camera_frustums,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LabViewProfile:
|
||||
result_id: str
|
||||
scene_settings: LabSceneProfile
|
||||
updated_at_utc: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if _RESULT_ID.fullmatch(self.result_id) is None or not self.updated_at_utc.endswith("Z"):
|
||||
raise LabViewProfileError("invalid LAB view profile identity")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": PROFILE_SCHEMA,
|
||||
"result_id": self.result_id,
|
||||
"scene_settings": self.scene_settings.as_dict(),
|
||||
"updated_at_utc": self.updated_at_utc,
|
||||
}
|
||||
|
||||
|
||||
class LabViewProfileStore:
|
||||
"""Mutable, atomic display state keyed by immutable LAB result identity."""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
root = root.expanduser().absolute()
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if root.is_symlink() or root.resolve() != root:
|
||||
raise LabViewProfileError("LAB view profile store must be a real directory")
|
||||
self.root = root
|
||||
|
||||
def _path(self, result_id: str) -> Path:
|
||||
if _RESULT_ID.fullmatch(result_id) is None:
|
||||
raise LabViewProfileError("invalid LAB result identity")
|
||||
digest = hashlib.sha256(result_id.encode("utf-8")).hexdigest()
|
||||
return self.root / f"{digest}.json"
|
||||
|
||||
def get(self, result_id: str) -> LabViewProfile | None:
|
||||
path = self._path(result_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024:
|
||||
raise LabViewProfileError("LAB view profile is unavailable")
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise LabViewProfileError("LAB view profile is unreadable") from exc
|
||||
return _decode(value, expected_result_id=result_id)
|
||||
|
||||
def save(self, profile: LabViewProfile) -> LabViewProfile:
|
||||
destination = self._path(profile.result_id)
|
||||
payload = (
|
||||
json.dumps(
|
||||
profile.as_dict(),
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
+ b"\n"
|
||||
)
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".view-profile-", dir=self.root)
|
||||
path = Path(temporary)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
path.chmod(0o600)
|
||||
path.replace(destination)
|
||||
return profile
|
||||
finally:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _decode(value: object, *, expected_result_id: str) -> LabViewProfile:
|
||||
keys = {"schema_version", "result_id", "scene_settings", "updated_at_utc"}
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) != keys
|
||||
or value.get("schema_version") != PROFILE_SCHEMA
|
||||
):
|
||||
raise LabViewProfileError("invalid LAB view profile document")
|
||||
row = cast(dict[str, object], value)
|
||||
if row["result_id"] != expected_result_id or not isinstance(row["updated_at_utc"], str):
|
||||
raise LabViewProfileError("LAB view profile identity changed")
|
||||
settings = row["scene_settings"]
|
||||
setting_keys = {
|
||||
"point_size",
|
||||
"accumulation_seconds",
|
||||
"color_mode",
|
||||
"palette",
|
||||
"show_grid",
|
||||
"show_labels",
|
||||
"show_camera_frustums",
|
||||
}
|
||||
if not isinstance(settings, dict) or set(settings) != setting_keys:
|
||||
raise LabViewProfileError("invalid LAB scene profile document")
|
||||
scene = cast(dict[str, object], settings)
|
||||
if (
|
||||
not isinstance(scene["point_size"], (int, float))
|
||||
or not isinstance(scene["accumulation_seconds"], (int, float))
|
||||
or not isinstance(scene["color_mode"], str)
|
||||
or not isinstance(scene["palette"], str)
|
||||
):
|
||||
raise LabViewProfileError("invalid LAB scene profile values")
|
||||
return LabViewProfile(
|
||||
result_id=expected_result_id,
|
||||
scene_settings=LabSceneProfile(
|
||||
point_size=float(scene["point_size"]),
|
||||
accumulation_seconds=float(scene["accumulation_seconds"]),
|
||||
color_mode=scene["color_mode"],
|
||||
palette=scene["palette"],
|
||||
show_grid=scene["show_grid"], # type: ignore[arg-type]
|
||||
show_labels=scene["show_labels"], # type: ignore[arg-type]
|
||||
show_camera_frustums=scene["show_camera_frustums"], # type: ignore[arg-type]
|
||||
),
|
||||
updated_at_utc=row["updated_at_utc"],
|
||||
)
|
||||
@@ -32,7 +32,8 @@ from typing import TYPE_CHECKING, Final, cast
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
|
||||
from k1link.compute.lidar_preparation import prepare_lidar_replay_pack_v2
|
||||
from k1link.compute.lidar_replay import LidarReplayPackV2
|
||||
from k1link.observatory.portable_result_contract import canonical_json
|
||||
from k1link.observatory.recorded_progress import report_recorded_progress
|
||||
from k1link.observatory.source_admission import (
|
||||
@@ -225,7 +226,7 @@ def materialize_m49_portable_source_from_worker_stage(
|
||||
if parent.is_symlink() or not parent.is_dir():
|
||||
raise M49PortableSourceError("portable M4.9 output parent is unsafe")
|
||||
try:
|
||||
lidar_pack_root = build_lidar_replay_pack_v2(
|
||||
lidar_pack_root = prepare_lidar_replay_pack_v2(
|
||||
root / "mqtt.raw.k1mqtt",
|
||||
parent / "lidar-replay-packs",
|
||||
session_id=job.source_session_id,
|
||||
@@ -538,7 +539,9 @@ def _materialize_stage(
|
||||
f"\t{anchor.session_seconds:.9f}\t-1\t0"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
"source-preparation",
|
||||
anchor.timeline_frame_index + 1,
|
||||
len(anchors),
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -600,7 +603,9 @@ def _materialize_stage(
|
||||
)
|
||||
available_slot += 1
|
||||
report_recorded_progress(
|
||||
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
|
||||
"source-preparation",
|
||||
anchor.timeline_frame_index + 1,
|
||||
len(anchors),
|
||||
)
|
||||
|
||||
if available_slot < 1:
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
"""Immutable, data-only AI compositions shared by Core and the installed Worker.
|
||||
|
||||
Selections contain no executable instructions. Installed module identities own
|
||||
their containers; the graph connects typed capabilities, never historical LABs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.portable_run_definitions import canonical_sha256
|
||||
|
||||
COMPOSITION_SCHEMA: Final = "missioncore.observatory-ai-composition/v1"
|
||||
MODULE_SCHEMA: Final = "missioncore.observatory-ai-module/v1"
|
||||
NODE_INPUT_SCHEMA: Final = "missioncore.observatory-ai-node-input/v1"
|
||||
GROUPS: Final = ("segmentation", "detection", "geometry", "range", "motion", "policy")
|
||||
_ID = re.compile(r"[a-z][a-z0-9.-]{1,95}\Z")
|
||||
_SHA = re.compile(r"[a-f0-9]{64}\Z")
|
||||
|
||||
|
||||
class CompositionError(ValueError):
|
||||
"""A selection, dependency or immutable identity is not admitted."""
|
||||
|
||||
|
||||
def canonical_bytes(value: object) -> bytes:
|
||||
try:
|
||||
return json.dumps(
|
||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
|
||||
).encode("utf-8")
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise CompositionError("composition must contain finite JSON data") from exc
|
||||
|
||||
|
||||
def require_digest(value: object) -> str:
|
||||
if not isinstance(value, str) or not _SHA.fullmatch(value):
|
||||
raise CompositionError("invalid immutable digest")
|
||||
return value
|
||||
|
||||
|
||||
def _identifier(value: object) -> str:
|
||||
if not isinstance(value, str) or not _ID.fullmatch(value):
|
||||
raise CompositionError("invalid module or capability identifier")
|
||||
return value
|
||||
|
||||
|
||||
def _object(value: object, keys: set[str]) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or set(value) != keys:
|
||||
raise CompositionError("unexpected composition fields")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModuleSpec:
|
||||
"""An installed version; all semantic and execution identities are sealed.
|
||||
|
||||
Source ports use the source.* namespace. Preparation modules are installed
|
||||
infrastructure and are added only when a selected module consumes a port.
|
||||
"""
|
||||
|
||||
module_id: str
|
||||
label: str
|
||||
group: str
|
||||
image_sha256: str
|
||||
implementation_sha256: str
|
||||
model_sha256: str | None
|
||||
contract_sha256: str
|
||||
requires: tuple[str, ...]
|
||||
provides: tuple[str, ...]
|
||||
optional_inputs: tuple[str, ...] = ()
|
||||
parameter_choices_json: bytes = b"{}"
|
||||
defaults_json: bytes = b"{}"
|
||||
state_policy: str = "stateless"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_identifier(self.module_id)
|
||||
if self.group not in (*GROUPS, "preparation"):
|
||||
raise CompositionError("unknown functional group")
|
||||
if not self.label or len(self.label) > 120:
|
||||
raise CompositionError("invalid module label")
|
||||
for digest in (self.image_sha256, self.implementation_sha256, self.contract_sha256):
|
||||
require_digest(digest)
|
||||
if self.model_sha256 is not None:
|
||||
require_digest(self.model_sha256)
|
||||
for ports in (self.requires, self.provides, self.optional_inputs):
|
||||
if tuple(sorted(set(ports))) != ports:
|
||||
raise CompositionError("module ports must be unique and sorted")
|
||||
for port in ports:
|
||||
_identifier(port)
|
||||
if not self.provides or any(port.startswith("source.") for port in self.provides):
|
||||
raise CompositionError("a module cannot provide raw source authority")
|
||||
if set(self.requires) & set(self.optional_inputs):
|
||||
raise CompositionError("required and optional ports overlap")
|
||||
if self.state_policy not in ("stateless", "causal-reset-at-source-start"):
|
||||
raise CompositionError("unqualified temporal state policy")
|
||||
choices = json.loads(self.parameter_choices_json)
|
||||
defaults = json.loads(self.defaults_json)
|
||||
if not isinstance(choices, dict) or not isinstance(defaults, dict):
|
||||
raise CompositionError("module parameters must be objects")
|
||||
if set(defaults) != set(choices):
|
||||
raise CompositionError("every parameter needs an explicit default")
|
||||
for key, values in choices.items():
|
||||
_identifier(key)
|
||||
if not isinstance(values, list) or not values:
|
||||
raise CompositionError("parameter choices must be finite nonempty lists")
|
||||
self.resolve_parameters(defaults)
|
||||
|
||||
def resolve_parameters(self, supplied: object) -> dict[str, object]:
|
||||
choices = cast(dict[str, list[object]], json.loads(self.parameter_choices_json))
|
||||
if not isinstance(supplied, dict) or set(supplied) - set(choices):
|
||||
raise CompositionError("parameter is not installed for this module")
|
||||
resolved: dict[str, object] = {
|
||||
**cast(dict[str, object], json.loads(self.defaults_json)),
|
||||
**supplied,
|
||||
}
|
||||
for key, value in resolved.items():
|
||||
# JSON identities distinguish true from 1 and reject nonfinite floats.
|
||||
if canonical_bytes(value) not in [canonical_bytes(item) for item in choices[key]]:
|
||||
raise CompositionError(f"unsupported value for {key}")
|
||||
return resolved
|
||||
|
||||
def identity_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": MODULE_SCHEMA,
|
||||
"module_id": self.module_id,
|
||||
"group": self.group,
|
||||
"image_sha256": self.image_sha256,
|
||||
"implementation_sha256": self.implementation_sha256,
|
||||
"model_sha256": self.model_sha256,
|
||||
"contract_sha256": self.contract_sha256,
|
||||
"requires": list(self.requires),
|
||||
"provides": list(self.provides),
|
||||
"optional_inputs": list(self.optional_inputs),
|
||||
"parameter_choices": json.loads(self.parameter_choices_json),
|
||||
"defaults": json.loads(self.defaults_json),
|
||||
"state_policy": self.state_policy,
|
||||
}
|
||||
|
||||
@property
|
||||
def sha256(self) -> str:
|
||||
return canonical_sha256(self.identity_document())
|
||||
|
||||
@property
|
||||
def docker_name(self) -> str:
|
||||
return f"ndc-mission-core-ai-module-{self.module_id}"
|
||||
|
||||
def docker_labels(self) -> dict[str, str]:
|
||||
return {
|
||||
"com.nodedc.product": "mission-core",
|
||||
"com.nodedc.stack": "observatory",
|
||||
"com.nodedc.role": "ai-module",
|
||||
"com.nodedc.managed-by": "mission-core-worker",
|
||||
"com.nodedc.module-id": self.module_id,
|
||||
"com.nodedc.module-sha256": self.sha256,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompositionNode:
|
||||
module: ModuleSpec
|
||||
parameters_json: bytes
|
||||
# Capability -> provider module ID, or source.* for authoritative source.
|
||||
inputs: tuple[tuple[str, str], ...]
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"module_id": self.module.module_id,
|
||||
"module_sha256": self.module.sha256,
|
||||
"parameters": json.loads(self.parameters_json),
|
||||
"inputs": dict(self.inputs),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompositionSpec:
|
||||
"""Source-independent graph, in deterministic topological execution order."""
|
||||
|
||||
nodes: tuple[CompositionNode, ...]
|
||||
|
||||
@property
|
||||
def source_capabilities(self) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
provider
|
||||
for node in self.nodes
|
||||
for _, provider in node.inputs
|
||||
if provider.startswith("source.")
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
@property
|
||||
def outputs(self) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
port
|
||||
for node in self.nodes
|
||||
if node.module.group != "preparation"
|
||||
for port in node.module.provides
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"nodes": [node.as_dict() for node in self.nodes],
|
||||
"source_capabilities": list(self.source_capabilities),
|
||||
"outputs": list(self.outputs),
|
||||
"execution": {"max_parallel_nodes": 1, "mode": "recorded-observation-only"},
|
||||
}
|
||||
|
||||
@property
|
||||
def sha256(self) -> str:
|
||||
return canonical_sha256(self.as_dict())
|
||||
|
||||
def selection_document(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": COMPOSITION_SCHEMA,
|
||||
"selections": [
|
||||
{
|
||||
"group": node.module.group,
|
||||
"module_id": node.module.module_id,
|
||||
"module_sha256": node.module.sha256,
|
||||
"parameters": json.loads(node.parameters_json),
|
||||
}
|
||||
for node in self.nodes
|
||||
if node.module.group != "preparation"
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class ModuleRegistry:
|
||||
def __init__(self, modules: tuple[ModuleSpec, ...]) -> None:
|
||||
self.modules = modules
|
||||
self._by_id = {module.module_id: module for module in modules}
|
||||
if len(self._by_id) != len(modules):
|
||||
raise CompositionError("duplicate installed module ID")
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> ModuleRegistry:
|
||||
candidate = path.expanduser().absolute()
|
||||
if (
|
||||
candidate.is_symlink()
|
||||
or not candidate.is_file()
|
||||
or candidate.stat().st_size > 1024 * 1024
|
||||
):
|
||||
raise CompositionError("AI module registry must be a bounded regular file")
|
||||
try:
|
||||
root = _object(json.loads(candidate.read_text()), {"schema_version", "modules"})
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise CompositionError("AI module registry is unreadable") from exc
|
||||
if root["schema_version"] != "missioncore.observatory-ai-module-registry/v1":
|
||||
raise CompositionError("unsupported AI module registry schema")
|
||||
if not isinstance(root["modules"], list):
|
||||
raise CompositionError("AI module registry rows must be an array")
|
||||
return cls(tuple(_module_from_dict(value) for value in root["modules"]))
|
||||
|
||||
def catalog(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-ai-module-catalog/v1",
|
||||
"groups": [
|
||||
{
|
||||
"group": group,
|
||||
"modules": [
|
||||
{
|
||||
"module_id": module.module_id,
|
||||
"module_sha256": module.sha256,
|
||||
"label": module.label,
|
||||
"docker_name": module.docker_name,
|
||||
"requires": list(module.requires),
|
||||
"provides": list(module.provides),
|
||||
"parameter_choices": json.loads(module.parameter_choices_json),
|
||||
"defaults": json.loads(module.defaults_json),
|
||||
}
|
||||
for module in self.modules
|
||||
if module.group == group
|
||||
],
|
||||
}
|
||||
for group in GROUPS
|
||||
],
|
||||
}
|
||||
|
||||
def compose(self, document: object) -> CompositionSpec:
|
||||
root = _object(document, {"schema_version", "selections"})
|
||||
if root["schema_version"] != COMPOSITION_SCHEMA:
|
||||
raise CompositionError("unsupported composition schema")
|
||||
selections = root["selections"]
|
||||
if not isinstance(selections, list) or not 1 <= len(selections) <= len(GROUPS):
|
||||
raise CompositionError("select at least one AI module")
|
||||
selected: dict[str, tuple[ModuleSpec, bytes]] = {}
|
||||
groups: set[str] = set()
|
||||
for raw in selections:
|
||||
row = _object(raw, {"group", "module_id", "module_sha256", "parameters"})
|
||||
module = self._by_id.get(_identifier(row["module_id"]))
|
||||
if module is None or module.sha256 != row["module_sha256"]:
|
||||
raise CompositionError("module version is not installed")
|
||||
if module.group != row["group"] or module.group not in GROUPS:
|
||||
raise CompositionError("module belongs to another functional group")
|
||||
if module.group in groups:
|
||||
raise CompositionError(f"select only one provider for {module.group}")
|
||||
groups.add(module.group)
|
||||
selected[module.module_id] = (
|
||||
module,
|
||||
canonical_bytes(module.resolve_parameters(row["parameters"])),
|
||||
)
|
||||
|
||||
def producers() -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
for module, _ in selected.values():
|
||||
for port in module.provides:
|
||||
if port in result:
|
||||
raise CompositionError(f"ambiguous provider for {port}")
|
||||
result[port] = module.module_id
|
||||
return result
|
||||
|
||||
# Only source preparation is implicit. Missing analytical dependencies
|
||||
# must be selected by the operator, never silently added to the LAB.
|
||||
while True:
|
||||
supplied = producers()
|
||||
missing = sorted(
|
||||
{
|
||||
port
|
||||
for module, _ in selected.values()
|
||||
for port in module.requires
|
||||
if not port.startswith("source.") and port not in supplied
|
||||
}
|
||||
)
|
||||
if not missing:
|
||||
break
|
||||
added = False
|
||||
for port in missing:
|
||||
candidates = [
|
||||
module
|
||||
for module in self.modules
|
||||
if module.group == "preparation" and port in module.provides
|
||||
]
|
||||
if len(candidates) != 1:
|
||||
raise CompositionError(f"select a module providing {port}")
|
||||
module = candidates[0]
|
||||
if module.module_id not in selected:
|
||||
selected[module.module_id] = (
|
||||
module,
|
||||
canonical_bytes(module.resolve_parameters({})),
|
||||
)
|
||||
added = True
|
||||
if not added:
|
||||
raise CompositionError("unresolved module dependencies")
|
||||
|
||||
supplied = producers()
|
||||
pending: dict[str, CompositionNode] = {}
|
||||
for module, parameters in selected.values():
|
||||
ports = (
|
||||
*module.requires,
|
||||
*(port for port in module.optional_inputs if port in supplied),
|
||||
)
|
||||
inputs = tuple(
|
||||
sorted(
|
||||
(port, port if port.startswith("source.") else supplied[port]) for port in ports
|
||||
)
|
||||
)
|
||||
pending[module.module_id] = CompositionNode(module, parameters, inputs)
|
||||
ordered: list[CompositionNode] = []
|
||||
emitted: set[str] = set()
|
||||
while pending:
|
||||
ready = sorted(
|
||||
key
|
||||
for key, node in pending.items()
|
||||
if all(
|
||||
provider.startswith("source.") or provider in emitted
|
||||
for _, provider in node.inputs
|
||||
)
|
||||
)
|
||||
if not ready:
|
||||
raise CompositionError("cyclic module dependencies")
|
||||
for key in ready:
|
||||
ordered.append(pending.pop(key))
|
||||
emitted.add(key)
|
||||
return CompositionSpec(tuple(ordered))
|
||||
|
||||
|
||||
def node_input_identity(
|
||||
node: CompositionNode,
|
||||
inputs: dict[str, str],
|
||||
*,
|
||||
state_context_sha256: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Exact node cache identity, independent of job and unrelated selections.
|
||||
|
||||
Each input digest seals bytes AND its I/O envelope (clock, calibration,
|
||||
preprocessing, cadence, precision, unavailable samples). Stateful nodes
|
||||
additionally bind their initialization and causal history.
|
||||
"""
|
||||
if set(inputs) != {port for port, _ in node.inputs}:
|
||||
raise CompositionError("node input capabilities disagree with the graph")
|
||||
for digest in inputs.values():
|
||||
require_digest(digest)
|
||||
if node.module.state_policy == "stateless":
|
||||
if state_context_sha256 is not None:
|
||||
raise CompositionError("stateless node cannot bind temporal state")
|
||||
else:
|
||||
require_digest(state_context_sha256)
|
||||
return {
|
||||
"schema_version": NODE_INPUT_SCHEMA,
|
||||
"module_sha256": node.module.sha256,
|
||||
"parameters": json.loads(node.parameters_json),
|
||||
"inputs": dict(sorted(inputs.items())),
|
||||
"state_policy": node.module.state_policy,
|
||||
"state_context_sha256": state_context_sha256,
|
||||
}
|
||||
|
||||
|
||||
def _module_from_dict(value: object) -> ModuleSpec:
|
||||
row = _object(
|
||||
value,
|
||||
{
|
||||
"module_id",
|
||||
"label",
|
||||
"group",
|
||||
"image_sha256",
|
||||
"implementation_sha256",
|
||||
"model_sha256",
|
||||
"contract_sha256",
|
||||
"requires",
|
||||
"provides",
|
||||
"optional_inputs",
|
||||
"parameter_choices",
|
||||
"defaults",
|
||||
"state_policy",
|
||||
},
|
||||
)
|
||||
arrays: dict[str, tuple[str, ...]] = {}
|
||||
for name in ("requires", "provides", "optional_inputs"):
|
||||
raw = row[name]
|
||||
if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw):
|
||||
raise CompositionError(f"module {name} must be an array of strings")
|
||||
arrays[name] = tuple(raw)
|
||||
return ModuleSpec(
|
||||
module_id=_identifier(row["module_id"]),
|
||||
label=row["label"] if isinstance(row["label"], str) else "",
|
||||
group=row["group"] if isinstance(row["group"], str) else "",
|
||||
image_sha256=require_digest(row["image_sha256"]),
|
||||
implementation_sha256=require_digest(row["implementation_sha256"]),
|
||||
model_sha256=None if row["model_sha256"] is None else require_digest(row["model_sha256"]),
|
||||
contract_sha256=require_digest(row["contract_sha256"]),
|
||||
requires=arrays["requires"],
|
||||
provides=arrays["provides"],
|
||||
optional_inputs=arrays["optional_inputs"],
|
||||
parameter_choices_json=canonical_bytes(row["parameter_choices"]),
|
||||
defaults_json=canonical_bytes(row["defaults"]),
|
||||
state_policy=row["state_policy"] if isinstance(row["state_policy"], str) else "",
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Durable immutable composition catalog owned by Core."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.observatory.modular_composition import (
|
||||
CompositionError,
|
||||
CompositionSpec,
|
||||
ModuleRegistry,
|
||||
canonical_bytes,
|
||||
)
|
||||
|
||||
|
||||
class ModularCompositionStore:
|
||||
def __init__(self, root: Path, registry: ModuleRegistry) -> None:
|
||||
root = root.expanduser().absolute()
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if root.is_symlink() or root.resolve() != root:
|
||||
raise CompositionError("composition store must be a real directory")
|
||||
self.root = root
|
||||
self.registry = registry
|
||||
|
||||
def save(self, selection: object) -> tuple[CompositionSpec, bool]:
|
||||
composition = self.registry.compose(selection)
|
||||
destination = self.root / f"{composition.sha256}.json"
|
||||
payload = canonical_bytes(composition.as_dict())
|
||||
if destination.exists():
|
||||
if destination.is_symlink() or destination.read_bytes() != payload:
|
||||
raise CompositionError("stored composition identity is damaged")
|
||||
return composition, False
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".composition-", dir=self.root)
|
||||
path = Path(temporary)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
path.chmod(0o444)
|
||||
try:
|
||||
path.rename(destination)
|
||||
except OSError:
|
||||
if destination.is_symlink() or destination.read_bytes() != payload:
|
||||
raise CompositionError("composition publication conflict") from None
|
||||
return composition, True
|
||||
finally:
|
||||
path.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Installed-package prepare and result steps for one independent AI module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.installed_lab_packages import (
|
||||
INSTALLED_LAB_PLAN_PATH,
|
||||
INSTALLED_LAB_RESULT_ROOT,
|
||||
INSTALLED_LAB_SOURCE_ROOT,
|
||||
INSTALLED_LAB_STEP_INPUT_ROOT,
|
||||
)
|
||||
from k1link.observatory.m49_portable_source import (
|
||||
M49_PORTABLE_STAGE_MANIFEST,
|
||||
materialize_m49_portable_source_from_worker_stage,
|
||||
)
|
||||
from k1link.observatory.modular_result import MODULAR_RESULT_KIND, MODULAR_RESULT_SCHEMA
|
||||
from k1link.observatory.portable_lab_v1_executor import (
|
||||
build_portable_ddrnet_effective_config,
|
||||
portable_lab_v1_source_input_from_document,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_local_runners import (
|
||||
PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||
)
|
||||
from k1link.observatory.portable_lab_v1_worker import (
|
||||
_SealedPackageJobView,
|
||||
materialize_recorded_camera_source_from_worker_stage,
|
||||
)
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
RESULT_PACKAGE_MANIFEST_NAME,
|
||||
PortableResultArtifact,
|
||||
PortableResultPackageManifest,
|
||||
canonical_json,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import (
|
||||
PortableRunDefinition,
|
||||
PortableRunDefinitionRegistry,
|
||||
canonical_sha256,
|
||||
)
|
||||
from k1link.observatory.portable_worker_runtime import PortableWorkerSourceStage
|
||||
from k1link.observatory.recorded_jobs import RecordedExecutorIdentity
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
MODULAR_PACKAGE_CONTRACT_SCHEMA: Final = (
|
||||
"missioncore.observatory-ai-module-installed-package-contract/v1"
|
||||
)
|
||||
_CONTRACT = Path("/opt/nodedc/package/contract.json")
|
||||
_DEFINITIONS = Path("/opt/nodedc/package/portable-run-definitions.json")
|
||||
_DDRNET_PROFILE = Path("/opt/nodedc/package/ddrnet-profile.json")
|
||||
_M49_PROFILE = Path("/opt/nodedc/package/m49-profile.json")
|
||||
_PREPARE = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "prepare"
|
||||
_MAX_DOCUMENT_BYTES: Final = 2 * 1024 * 1024
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
|
||||
_ARTIFACTS: Final = {
|
||||
"ddrnet": (
|
||||
("ddrnet-decode-repair", "decode-repair.json", "application/json"),
|
||||
("ddrnet-result-document", "result.json", "application/json"),
|
||||
("ddrnet-semantic-mask-archive", "semantic-masks.zip", "application/zip"),
|
||||
),
|
||||
"eomt": (
|
||||
("eomt-decode-repair", "decode-repair.json", "application/json"),
|
||||
("eomt-panoptic-frame-metadata", "frames.jsonl", "application/x-ndjson"),
|
||||
("eomt-gpu-telemetry", "gpu-telemetry.jsonl", "application/x-ndjson"),
|
||||
("eomt-panoptic-mask-archive", "masks.tar.gz", "application/gzip"),
|
||||
("eomt-overlay-video", "perception.mp4", "video/mp4"),
|
||||
("eomt-result-document", "result.json", "application/json"),
|
||||
("eomt-run-report", "run-report.json", "application/json"),
|
||||
("eomt-source-frame-manifest", "source-frames.json", "application/json"),
|
||||
),
|
||||
"rf-detr": (
|
||||
("rf-detr-frame-detections", "detections.jsonl", "application/x-ndjson"),
|
||||
("rf-detr-result-document", "result.json", "application/json"),
|
||||
),
|
||||
"object-distance": (
|
||||
("rf-detr-frame-detections", "detections.jsonl", "application/x-ndjson"),
|
||||
("rf-detr-result-document", "result.json", "application/json"),
|
||||
(
|
||||
"object-distance-frame-observations",
|
||||
"object-distances.jsonl",
|
||||
"application/x-ndjson",
|
||||
),
|
||||
("object-distance-result-document", "result.json", "application/json"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class ModularPackageStepError(RuntimeError):
|
||||
"""The selected installed module package changed or is incomplete."""
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
arguments = tuple(sys.argv[1:] if argv is None else argv)
|
||||
if arguments == ("prepare",):
|
||||
prepare()
|
||||
return 0
|
||||
if arguments == ("assemble",):
|
||||
assemble()
|
||||
return 0
|
||||
raise ModularPackageStepError("AI-module package step is not allowlisted")
|
||||
|
||||
|
||||
def prepare() -> None:
|
||||
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "AI-module prepare output")
|
||||
contract = _load_contract()
|
||||
runtime_plan = _runtime_plan()
|
||||
definition = _definition(runtime_plan)
|
||||
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
|
||||
module = _object(contract["module"], "AI module")
|
||||
module_id = _identifier(module["module_id"], "module id")
|
||||
stage = PortableWorkerSourceStage(
|
||||
root=_real_directory(Path(INSTALLED_LAB_SOURCE_ROOT), "AI-module source"),
|
||||
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "source bundle"),
|
||||
source_capability_manifest_sha256=_digest(
|
||||
runtime_plan["source_capability_manifest_sha256"], "source capability"
|
||||
),
|
||||
source_adapter_sha256=_digest(runtime_plan["source_adapter_sha256"], "source adapter"),
|
||||
)
|
||||
materialized = materialize_recorded_camera_source_from_worker_stage(
|
||||
worker_stage=stage,
|
||||
job=job,
|
||||
definition=definition,
|
||||
output_parent=output,
|
||||
)
|
||||
camera_target = output / "camera-job"
|
||||
camera_stage_parent = materialized.camera_job_root.parent
|
||||
os.replace(materialized.camera_job_root, camera_target)
|
||||
camera_stage_parent.rmdir()
|
||||
materialized.root.rmdir()
|
||||
source = materialized.descriptor.as_dict()
|
||||
_write(output / "source-input.json", source)
|
||||
if module_id == "object-distance":
|
||||
spatial_root = output / "spatial-source"
|
||||
m49 = materialize_m49_portable_source_from_worker_stage(
|
||||
worker_stage=stage,
|
||||
job=job,
|
||||
profile_path=_M49_PROFILE,
|
||||
output_parent=spatial_root,
|
||||
)
|
||||
manifest = _load_object(
|
||||
m49.root / M49_PORTABLE_STAGE_MANIFEST,
|
||||
"object-distance spatial source",
|
||||
)
|
||||
identity = _object(manifest.get("identity"), "object-distance spatial identity")
|
||||
lidar = _object(identity.get("lidar_replay"), "object-distance LiDAR identity")
|
||||
pack_id = _identifier(lidar.get("pack_id"), "object-distance LiDAR pack id")
|
||||
pack = _real_directory(
|
||||
spatial_root / "lidar-replay-packs" / pack_id,
|
||||
"object-distance LiDAR pack",
|
||||
)
|
||||
os.replace(pack, output / "lidar-pack")
|
||||
os.replace(m49.root, output / "m49-source")
|
||||
shutil.rmtree(spatial_root)
|
||||
plan_sha = canonical_sha256(
|
||||
{
|
||||
"schema_version": "missioncore.observatory-ai-module-plan/v1",
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"module_id": module_id,
|
||||
"module_sha256": _digest(module["module_sha256"], "module identity"),
|
||||
"source_input_sha256": materialized.descriptor.identity_sha256,
|
||||
}
|
||||
)
|
||||
assets = _object(contract["component_assets"], "component assets")
|
||||
images = _object(contract["component_images"], "component images")
|
||||
_write(
|
||||
output / "camera-source-request.json",
|
||||
_component_request(
|
||||
component="camera-source",
|
||||
image_sha=_digest(images["camera-source"], "camera image"),
|
||||
source=source,
|
||||
plan_sha=plan_sha,
|
||||
definition=definition,
|
||||
release_sha=_digest(runtime_plan["candidate_sha256"], "runtime candidate"),
|
||||
assets=_asset_rows(assets["camera-source"], "camera-source"),
|
||||
ddrnet_config_sha=None,
|
||||
),
|
||||
)
|
||||
ddrnet_config_sha: str | None = None
|
||||
if module_id == "ddrnet":
|
||||
profile = _load_object(_DDRNET_PROFILE, "DDRNet profile")
|
||||
effective = build_portable_ddrnet_effective_config(
|
||||
profile,
|
||||
source=materialized.descriptor,
|
||||
)
|
||||
_write(output / "effective-ddrnet-config.json", effective)
|
||||
ddrnet_config_sha = canonical_sha256(effective)
|
||||
_write(
|
||||
output / f"{module_id}-request.json",
|
||||
_component_request(
|
||||
component=module_id,
|
||||
image_sha=_digest(images[module_id], "module image"),
|
||||
source=source,
|
||||
plan_sha=plan_sha,
|
||||
definition=definition,
|
||||
release_sha=_digest(runtime_plan["candidate_sha256"], "runtime candidate"),
|
||||
assets=_asset_rows(assets[module_id], module_id),
|
||||
ddrnet_config_sha=ddrnet_config_sha,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def assemble() -> None:
|
||||
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "AI-module result output")
|
||||
artifacts_root = output / "artifacts"
|
||||
artifacts_root.mkdir(mode=0o700)
|
||||
contract = _load_contract()
|
||||
runtime_plan = _runtime_plan()
|
||||
definition = _definition(runtime_plan)
|
||||
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
|
||||
source_input = portable_lab_v1_source_input_from_document(
|
||||
_load_object(_PREPARE / "source-input.json", "AI-module source input")
|
||||
)
|
||||
module = _object(contract["module"], "AI module")
|
||||
module_id = _identifier(module["module_id"], "module id")
|
||||
module_root = _real_directory(
|
||||
Path(INSTALLED_LAB_STEP_INPUT_ROOT) / module_id, "AI-module output"
|
||||
)
|
||||
component_result = _regular_file(module_root / "result.json", module_root, "module result")
|
||||
component_result_sha = _sha256(component_result)
|
||||
artifacts: list[PortableResultArtifact] = []
|
||||
for role, name, media_type in _ARTIFACTS[module_id]:
|
||||
artifact_root = (
|
||||
_real_directory(
|
||||
Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "rf-detr",
|
||||
"RF-DETR dependency output",
|
||||
)
|
||||
if module_id == "object-distance" and role.startswith("rf-detr-")
|
||||
else module_root
|
||||
)
|
||||
source_path = _regular_file(artifact_root / name, artifact_root, role)
|
||||
artifact_name = f"{role}{source_path.suffix}"
|
||||
if name.endswith(".tar.gz"):
|
||||
artifact_name = f"{role}.tar.gz"
|
||||
target_name = f"artifacts/{artifact_name}"
|
||||
target = artifacts_root / artifact_name
|
||||
_copy(source_path, target)
|
||||
artifacts.append(_artifact(role, target_name, media_type, target))
|
||||
source = {
|
||||
"session_id": job.source_session_id,
|
||||
"catalog_sha256": job.source_catalog_sha256,
|
||||
"bundle_sha256": job.source_bundle_sha256,
|
||||
"capability_manifest_sha256": job.source_capability_manifest_sha256,
|
||||
"camera_input_sha256": source_input.camera_input_sha256,
|
||||
"frame_count": source_input.frame_count,
|
||||
"timeline_start_seconds": source_input.timeline_start_seconds,
|
||||
"timeline_end_seconds": source_input.timeline_end_seconds,
|
||||
}
|
||||
module_view = {
|
||||
"module_id": module_id,
|
||||
"label": _text(module["label"], "module label"),
|
||||
"module_sha256": _digest(module["module_sha256"], "module identity"),
|
||||
"image_sha256": _digest(module["image_sha256"], "module image"),
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"component_result_sha256": component_result_sha,
|
||||
}
|
||||
identity = {
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"source_bundle_sha256": job.source_bundle_sha256,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"module_id": module_id,
|
||||
"component_result_sha256": component_result_sha,
|
||||
}
|
||||
identity_sha = canonical_sha256(identity)
|
||||
result_id = f"ai-layer-{module_id}-{identity_sha}"
|
||||
result_document = {
|
||||
"schema_version": MODULAR_RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"result_kind": MODULAR_RESULT_KIND,
|
||||
"identity": identity,
|
||||
"identity_sha256": identity_sha,
|
||||
"source": source,
|
||||
"module": module_view,
|
||||
"artifacts": [item.as_dict() for item in sorted(artifacts, key=lambda item: item.role)],
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
result_path = artifacts_root / "result.json"
|
||||
_write(result_path, result_document)
|
||||
artifacts.append(
|
||||
_artifact(
|
||||
RESULT_DOCUMENT_ROLE,
|
||||
"artifacts/result.json",
|
||||
"application/json",
|
||||
result_path,
|
||||
)
|
||||
)
|
||||
manifest = PortableResultPackageManifest.create(
|
||||
job=_SealedPackageJobView.from_job(job),
|
||||
definition=definition,
|
||||
result_id=result_id,
|
||||
created_at_utc=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||||
artifacts=artifacts,
|
||||
)
|
||||
(output / RESULT_PACKAGE_MANIFEST_NAME).write_bytes(manifest.canonical_bytes)
|
||||
os.chmod(output / RESULT_PACKAGE_MANIFEST_NAME, 0o400)
|
||||
|
||||
|
||||
def _component_request(
|
||||
*,
|
||||
component: str,
|
||||
image_sha: str,
|
||||
source: Mapping[str, object],
|
||||
plan_sha: str,
|
||||
definition: PortableRunDefinition,
|
||||
release_sha: str,
|
||||
assets: list[dict[str, object]],
|
||||
ddrnet_config_sha: str | None,
|
||||
) -> dict[str, object]:
|
||||
prepared = f"{INSTALLED_LAB_STEP_INPUT_ROOT}/prepare"
|
||||
return {
|
||||
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||
"component": component,
|
||||
"component_image_sha256": image_sha,
|
||||
"plan_sha256": plan_sha,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"release_candidate_sha256": release_sha,
|
||||
"source": dict(source),
|
||||
"paths": {
|
||||
"camera_job_root": f"{prepared}/camera-job",
|
||||
"request": f"{prepared}/{component}-request.json",
|
||||
"output_root": INSTALLED_LAB_RESULT_ROOT,
|
||||
"effective_ddrnet_config": (
|
||||
f"{prepared}/effective-ddrnet-config.json" if component == "ddrnet" else None
|
||||
),
|
||||
"eomt_result_root": (
|
||||
f"{INSTALLED_LAB_STEP_INPUT_ROOT}/camera-source" if component == "ddrnet" else None
|
||||
),
|
||||
"decoded_frames_root": (
|
||||
f"{INSTALLED_LAB_RESULT_ROOT}/source-frames"
|
||||
if component == "camera-source"
|
||||
else f"{INSTALLED_LAB_STEP_INPUT_ROOT}/camera-source/source-frames"
|
||||
),
|
||||
},
|
||||
"effective_ddrnet_config_sha256": ddrnet_config_sha,
|
||||
"assets": assets,
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
|
||||
|
||||
def _sealed_job(
|
||||
runtime_plan: Mapping[str, object],
|
||||
*,
|
||||
definition: PortableRunDefinition,
|
||||
contract: Mapping[str, object],
|
||||
) -> SealedObservatoryRecordedJob:
|
||||
executor = _object(contract["executor"], "package executor")
|
||||
identity = RecordedExecutorIdentity(
|
||||
release_sha256=_digest(executor["release_sha256"], "executor release"),
|
||||
image_sha256=_digest(executor["image_sha256"], "executor image"),
|
||||
model_manifest_sha256=definition.model_manifest_sha256,
|
||||
resource_profile_sha256=definition.resource_profile.profile_sha256,
|
||||
)
|
||||
return SealedObservatoryRecordedJob(
|
||||
job_id=_text(runtime_plan["job_id"], "job id"),
|
||||
request_sha256=_digest(runtime_plan["request_sha256"], "request"),
|
||||
identity_sha256=_digest(runtime_plan["identity_sha256"], "job identity"),
|
||||
submission_receipt_sha256=_digest(runtime_plan["submission_receipt_sha256"], "receipt"),
|
||||
source_session_id=_text(runtime_plan["source_session_id"], "source session"),
|
||||
source_catalog_sha256=_digest(runtime_plan["source_catalog_sha256"], "catalog"),
|
||||
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "bundle"),
|
||||
source_capability_manifest_sha256=_digest(
|
||||
runtime_plan["source_capability_manifest_sha256"], "capability"
|
||||
),
|
||||
source_adapter_id=_identifier(runtime_plan["source_adapter_id"], "adapter id"),
|
||||
source_adapter_version=_positive_int(
|
||||
runtime_plan["source_adapter_version"], "adapter version"
|
||||
),
|
||||
source_adapter_sha256=_digest(runtime_plan["source_adapter_sha256"], "adapter"),
|
||||
setup_id=definition.setup_id,
|
||||
definition_id=definition.definition_id,
|
||||
definition_version=definition.version,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
executor_release_id=_identifier(executor["release_id"], "executor release id"),
|
||||
executor_identity=identity,
|
||||
model_release_ids=definition.learned_models,
|
||||
resource_profile_id=definition.resource_profile.profile_id,
|
||||
checkpoint_policy=definition.resource_profile.checkpoint_policy,
|
||||
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
|
||||
claim_generation=_positive_int(runtime_plan["claim_generation"], "claim generation"),
|
||||
claim_claimed_at_utc=None,
|
||||
claim_expires_at_utc=None,
|
||||
claim_heartbeat_at_utc=None,
|
||||
claim_renewal_count=0,
|
||||
restart_from_zero=False,
|
||||
)
|
||||
|
||||
|
||||
def _runtime_plan() -> dict[str, object]:
|
||||
outer = _load_object(Path(INSTALLED_LAB_PLAN_PATH), "installed AI-module run plan")
|
||||
if (
|
||||
outer.get("schema_version") != "missioncore.observatory-installed-lab-run-plan/v1"
|
||||
or outer.get("authority") != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise ModularPackageStepError("installed AI-module run plan changed")
|
||||
return _object(outer.get("runtime_plan"), "portable runtime plan")
|
||||
|
||||
|
||||
def _definition(runtime_plan: Mapping[str, object]) -> PortableRunDefinition:
|
||||
return PortableRunDefinitionRegistry.from_file(_DEFINITIONS).resolve(
|
||||
_identifier(runtime_plan["setup_id"], "setup id"),
|
||||
_digest(runtime_plan["definition_sha256"], "definition"),
|
||||
)
|
||||
|
||||
|
||||
def _load_contract() -> dict[str, object]:
|
||||
value = _load_object(_CONTRACT, "AI-module package contract")
|
||||
if (
|
||||
value.get("schema_version") != MODULAR_PACKAGE_CONTRACT_SCHEMA
|
||||
or value.get("authority") != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise ModularPackageStepError("AI-module package contract changed")
|
||||
return value
|
||||
|
||||
|
||||
def _asset_rows(value: object, component: str) -> list[dict[str, object]]:
|
||||
if not isinstance(value, list):
|
||||
raise ModularPackageStepError(f"{component} assets are not an array")
|
||||
rows = [dict(_object(row, f"{component} asset")) for row in value]
|
||||
ids = tuple(cast(str, row.get("asset_id")) for row in rows)
|
||||
if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)):
|
||||
raise ModularPackageStepError(f"{component} assets are not canonical")
|
||||
return rows
|
||||
|
||||
|
||||
def _artifact(role: str, relative: str, media: str, path: Path) -> PortableResultArtifact:
|
||||
return PortableResultArtifact(role, relative, media, path.stat().st_size, _sha256(path))
|
||||
|
||||
|
||||
def _copy(source: Path, target: Path) -> None:
|
||||
with source.open("rb") as reader, target.open("xb") as writer:
|
||||
shutil.copyfileobj(reader, writer, 1024 * 1024)
|
||||
os.chmod(target, 0o400)
|
||||
if target.stat().st_size != source.stat().st_size or _sha256(target) != _sha256(source):
|
||||
raise ModularPackageStepError("AI-module artifact copy changed")
|
||||
|
||||
|
||||
def _write(path: Path, value: Mapping[str, object]) -> None:
|
||||
path.write_bytes(canonical_json(value))
|
||||
os.chmod(path, 0o400)
|
||||
|
||||
|
||||
def _load_object(path: Path, label: str) -> dict[str, object]:
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not path.is_file()
|
||||
or not 0 < path.stat().st_size <= _MAX_DOCUMENT_BYTES
|
||||
):
|
||||
raise ModularPackageStepError(f"{label} is unavailable")
|
||||
try:
|
||||
return _object(json.loads(path.read_bytes()), label)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ModularPackageStepError(f"{label} is invalid JSON") from exc
|
||||
|
||||
|
||||
def _regular_file(path: Path, root: Path, label: str) -> Path:
|
||||
if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
|
||||
raise ModularPackageStepError(f"{label} is unavailable")
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def _empty_directory(path: Path, label: str) -> Path:
|
||||
root = _real_directory(path, label)
|
||||
if any(root.iterdir()):
|
||||
raise ModularPackageStepError(f"{label} is not empty")
|
||||
return root
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise ModularPackageStepError(f"{label} is unavailable")
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise ModularPackageStepError(f"{label} is invalid")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _text(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ModularPackageStepError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _identifier(value: object, label: str) -> str:
|
||||
text = _text(value, label)
|
||||
if _IDENTIFIER.fullmatch(text) is None:
|
||||
raise ModularPackageStepError(f"{label} is invalid")
|
||||
return text
|
||||
|
||||
|
||||
def _digest(value: object, label: str) -> str:
|
||||
text = _text(value, label)
|
||||
if _SHA256.fullmatch(text) is None:
|
||||
raise ModularPackageStepError(f"{label} is invalid")
|
||||
return text
|
||||
|
||||
|
||||
def _positive_int(value: object, label: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
raise ModularPackageStepError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except ModularPackageStepError as exc:
|
||||
print(f"installed AI-module package rejected: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2) from exc
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Sealed Worker-local node results; source CAS and final Core LABs are separate.
|
||||
|
||||
Entries are published by atomic directory rename only after all regular files
|
||||
are hashed. Readers validate bytes, not existence. A damaged entry is a miss;
|
||||
original evidence is never overwritten or removed by a reader.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import cast
|
||||
|
||||
from k1link.observatory.modular_composition import (
|
||||
CompositionError,
|
||||
canonical_bytes,
|
||||
require_digest,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import canonical_sha256
|
||||
|
||||
NODE_RESULT_SCHEMA = "missioncore.observatory-ai-node-result/v1"
|
||||
_MAX_MANIFEST_BYTES = 4 * 1024 * 1024
|
||||
_MAX_FILES = 20_000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SealedNodeResult:
|
||||
root: Path
|
||||
input_sha256: str
|
||||
result_sha256: str
|
||||
manifest_json: bytes
|
||||
|
||||
@property
|
||||
def manifest(self) -> dict[str, object]:
|
||||
value = json.loads(self.manifest_json)
|
||||
if not isinstance(value, dict):
|
||||
raise CompositionError("node manifest must be an object")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> tuple[int, str]:
|
||||
mode = path.lstat().st_mode
|
||||
if not stat.S_ISREG(mode):
|
||||
raise CompositionError("node result must contain only regular files")
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
def _files(root: Path) -> list[dict[str, object]]:
|
||||
entries: list[dict[str, object]] = []
|
||||
for path in sorted(root.rglob("*")):
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISDIR(mode):
|
||||
continue
|
||||
size, digest = _hash_file(path)
|
||||
entries.append(
|
||||
{"path": path.relative_to(root).as_posix(), "byte_length": size, "sha256": digest}
|
||||
)
|
||||
if len(entries) > _MAX_FILES:
|
||||
raise CompositionError("node result exceeds the file limit")
|
||||
if not entries:
|
||||
raise CompositionError("empty node output cannot be sealed")
|
||||
return entries
|
||||
|
||||
|
||||
class ModularNodeCache:
|
||||
def __init__(self, root: Path) -> None:
|
||||
root = root.expanduser().absolute()
|
||||
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if root.is_symlink() or not root.is_dir() or root.resolve() != root:
|
||||
raise CompositionError("node cache root must be a real directory")
|
||||
self.root = root
|
||||
|
||||
def lookup(self, input_identity: dict[str, object]) -> SealedNodeResult | None:
|
||||
input_sha256 = canonical_sha256(input_identity)
|
||||
entry = self.root / input_sha256
|
||||
try:
|
||||
if entry.is_symlink() or not entry.is_dir():
|
||||
return None
|
||||
manifest_path = entry / "manifest.json"
|
||||
if (
|
||||
not stat.S_ISREG(manifest_path.lstat().st_mode)
|
||||
or manifest_path.stat().st_size > _MAX_MANIFEST_BYTES
|
||||
):
|
||||
return None
|
||||
payload = manifest_path.read_bytes()
|
||||
manifest = cast(dict[str, object], json.loads(payload))
|
||||
if set(manifest) != {
|
||||
"schema_version",
|
||||
"input_identity",
|
||||
"input_sha256",
|
||||
"outputs",
|
||||
"metadata",
|
||||
"result_sha256",
|
||||
}:
|
||||
return None
|
||||
identity = {key: value for key, value in manifest.items() if key != "result_sha256"}
|
||||
if (
|
||||
manifest["schema_version"] != NODE_RESULT_SCHEMA
|
||||
or manifest["input_identity"] != input_identity
|
||||
or manifest["input_sha256"] != input_sha256
|
||||
or manifest["result_sha256"] != canonical_sha256(identity)
|
||||
or payload != canonical_bytes(manifest)
|
||||
):
|
||||
return None
|
||||
data = entry / "data"
|
||||
if data.is_symlink() or not data.is_dir() or _files(data) != manifest["outputs"]:
|
||||
return None
|
||||
return SealedNodeResult(data, input_sha256, manifest["result_sha256"], payload)
|
||||
except (OSError, ValueError, TypeError, KeyError):
|
||||
return None
|
||||
|
||||
def seal(
|
||||
self,
|
||||
input_identity: dict[str, object],
|
||||
output_root: Path,
|
||||
*,
|
||||
metadata: dict[str, object],
|
||||
) -> SealedNodeResult:
|
||||
existing = self.lookup(input_identity)
|
||||
if existing is not None:
|
||||
return existing
|
||||
input_sha256 = canonical_sha256(input_identity)
|
||||
require_digest(input_sha256)
|
||||
destination = self.root / input_sha256
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise CompositionError("damaged node cache entry requires explicit repair")
|
||||
output_root = output_root.absolute()
|
||||
if output_root.is_symlink() or output_root.resolve() != output_root:
|
||||
raise CompositionError("node output root must be a real directory")
|
||||
entries = _files(output_root)
|
||||
identity = {
|
||||
"schema_version": NODE_RESULT_SCHEMA,
|
||||
"input_identity": input_identity,
|
||||
"input_sha256": input_sha256,
|
||||
"outputs": entries,
|
||||
"metadata": metadata,
|
||||
}
|
||||
result_sha256 = canonical_sha256(identity)
|
||||
payload = canonical_bytes({**identity, "result_sha256": result_sha256})
|
||||
if len(payload) > _MAX_MANIFEST_BYTES:
|
||||
raise CompositionError("node manifest exceeds the size limit")
|
||||
stage = Path(tempfile.mkdtemp(prefix=".node-seal-", dir=self.root))
|
||||
try:
|
||||
data = stage / "data"
|
||||
data.mkdir(mode=0o700)
|
||||
for entry in entries:
|
||||
relative = PurePosixPath(cast(str, entry["path"]))
|
||||
target = data.joinpath(*relative.parts)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Copy: chmod on a hardlink would mutate the producer's files,
|
||||
# and a surviving producer could corrupt a supposedly sealed entry.
|
||||
with (output_root / relative).open("rb") as source, target.open("xb") as out:
|
||||
shutil.copyfileobj(source, out, 1024 * 1024)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
target.chmod(0o444)
|
||||
if _files(data) != entries:
|
||||
raise CompositionError("node output changed while sealing")
|
||||
with (stage / "manifest.json").open("xb") as out:
|
||||
out.write(payload)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
(stage / "manifest.json").chmod(0o444)
|
||||
try:
|
||||
stage.rename(destination)
|
||||
except OSError:
|
||||
# Another process may have atomically won this exact identity.
|
||||
winner = self.lookup(input_identity)
|
||||
if winner is None or winner.result_sha256 != result_sha256:
|
||||
raise CompositionError("node seal conflicts with an existing result") from None
|
||||
return winner
|
||||
_fsync_directory(self.root)
|
||||
return SealedNodeResult(destination / "data", input_sha256, result_sha256, payload)
|
||||
finally:
|
||||
if stage.exists():
|
||||
shutil.rmtree(stage)
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Result contract for one independently selected Observatory AI module."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from k1link.observatory.portable_result_contract import (
|
||||
OBSERVATION_ONLY_AUTHORITY,
|
||||
PortableResultArtifact,
|
||||
PortableResultPackageIntegrityError,
|
||||
PortableResultValidationContext,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import canonical_sha256
|
||||
|
||||
MODULAR_RESULT_SCHEMA: Final = "missioncore.recorded-ai-layer-review/v1"
|
||||
MODULAR_RESULT_KIND: Final = "recorded-ai-layer-review"
|
||||
MODULAR_RESULT_CONTRACT_SHA256: Final = (
|
||||
"2316c027ed6014d3ca5632e38de3bbed448ca79e7243fb300510565b21d56305"
|
||||
)
|
||||
MODULE_BY_SETUP: Final = {
|
||||
"ai-segmentation-ddrnet-v1": "ddrnet",
|
||||
"ai-segmentation-eomt-v1": "eomt",
|
||||
"ai-detection-rf-detr-v1": "rf-detr",
|
||||
"ai-range-object-distance-v1": "object-distance",
|
||||
}
|
||||
_EXPECTED_ARTIFACT_ROLES: Final = {
|
||||
"ddrnet": {
|
||||
"ddrnet-decode-repair",
|
||||
"ddrnet-result-document",
|
||||
"ddrnet-semantic-mask-archive",
|
||||
},
|
||||
"eomt": {
|
||||
"eomt-decode-repair",
|
||||
"eomt-panoptic-frame-metadata",
|
||||
"eomt-gpu-telemetry",
|
||||
"eomt-panoptic-mask-archive",
|
||||
"eomt-overlay-video",
|
||||
"eomt-result-document",
|
||||
"eomt-run-report",
|
||||
"eomt-source-frame-manifest",
|
||||
},
|
||||
"rf-detr": {"rf-detr-frame-detections", "rf-detr-result-document"},
|
||||
"object-distance": {
|
||||
"object-distance-frame-observations",
|
||||
"object-distance-result-document",
|
||||
"rf-detr-frame-detections",
|
||||
"rf-detr-result-document",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def validate_modular_result(context: PortableResultValidationContext) -> None:
|
||||
"""Re-bind a packaged module result to its exact job and artifacts."""
|
||||
|
||||
definition = context.definition
|
||||
module_id = MODULE_BY_SETUP.get(definition.setup_id)
|
||||
document = context.result_document
|
||||
if (
|
||||
module_id is None
|
||||
or definition.result_contract.contract_sha256 != MODULAR_RESULT_CONTRACT_SHA256
|
||||
or definition.result_contract.result_schema != MODULAR_RESULT_SCHEMA
|
||||
or definition.result_contract.result_kind != MODULAR_RESULT_KIND
|
||||
or set(document)
|
||||
!= {
|
||||
"schema_version",
|
||||
"result_id",
|
||||
"result_kind",
|
||||
"identity",
|
||||
"identity_sha256",
|
||||
"source",
|
||||
"module",
|
||||
"artifacts",
|
||||
"authority",
|
||||
}
|
||||
or document.get("schema_version") != MODULAR_RESULT_SCHEMA
|
||||
or document.get("result_kind") != MODULAR_RESULT_KIND
|
||||
or document.get("authority") != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("AI-layer result envelope changed")
|
||||
identity = _object(document.get("identity"), "AI-layer identity")
|
||||
identity_sha = document.get("identity_sha256")
|
||||
if (
|
||||
not isinstance(identity_sha, str)
|
||||
or canonical_sha256(identity) != identity_sha
|
||||
or document.get("result_id") != f"ai-layer-{module_id}-{identity_sha}"
|
||||
or document.get("result_id") != context.job.result_id
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("AI-layer result identity changed")
|
||||
source = _object(document.get("source"), "AI-layer source")
|
||||
module = _object(document.get("module"), "AI-layer module")
|
||||
expected_source = {
|
||||
"session_id": context.job.source_session_id,
|
||||
"catalog_sha256": context.job.source_catalog_sha256,
|
||||
"bundle_sha256": context.job.source_bundle_sha256,
|
||||
"capability_manifest_sha256": context.job.source_capability_manifest_sha256,
|
||||
}
|
||||
if (
|
||||
any(source.get(key) != value for key, value in expected_source.items())
|
||||
or module.get("module_id") != module_id
|
||||
or module.get("definition_sha256") != definition.definition_sha256
|
||||
or identity
|
||||
!= {
|
||||
"job_identity_sha256": context.job.identity_sha256,
|
||||
"source_bundle_sha256": context.job.source_bundle_sha256,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"module_id": module_id,
|
||||
"component_result_sha256": module.get("component_result_sha256"),
|
||||
}
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("AI-layer provenance changed")
|
||||
declared_value = document.get("artifacts")
|
||||
if not isinstance(declared_value, list):
|
||||
raise PortableResultPackageIntegrityError("AI-layer artifacts are invalid")
|
||||
declared = tuple(_artifact(row) for row in declared_value)
|
||||
if (
|
||||
tuple(item.role for item in declared) != tuple(sorted(item.role for item in declared))
|
||||
or {item.role for item in declared} != _EXPECTED_ARTIFACT_ROLES[module_id]
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("AI-layer artifacts are not canonical")
|
||||
packaged = {item.role: item for item in context.manifest.artifacts}
|
||||
if set(packaged) != {"result-document", *(item.role for item in declared)}:
|
||||
raise PortableResultPackageIntegrityError("AI-layer package artifacts changed")
|
||||
for artifact in declared:
|
||||
path = context.artifact_paths.get(artifact.role)
|
||||
if (
|
||||
packaged.get(artifact.role) != artifact
|
||||
or path is None
|
||||
or _sha256(path) != artifact.sha256
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("AI-layer artifact content changed")
|
||||
result_role = f"{module_id}-result-document"
|
||||
component_path = context.artifact_paths.get(result_role)
|
||||
if component_path is None or _sha256(component_path) != module.get("component_result_sha256"):
|
||||
raise PortableResultPackageIntegrityError("AI-layer component result changed")
|
||||
try:
|
||||
component = json.loads(component_path.read_bytes())
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultPackageIntegrityError("AI-layer component result is invalid") from exc
|
||||
_validate_component(module_id, _object(component, "component result"), source, packaged)
|
||||
if module_id == "object-distance":
|
||||
dependency_path = context.artifact_paths.get("rf-detr-result-document")
|
||||
if dependency_path is None:
|
||||
raise PortableResultPackageIntegrityError("RF-DETR dependency result is missing")
|
||||
try:
|
||||
dependency = _object(
|
||||
json.loads(dependency_path.read_bytes()), "RF-DETR dependency result"
|
||||
)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise PortableResultPackageIntegrityError(
|
||||
"RF-DETR dependency result is invalid"
|
||||
) from exc
|
||||
_validate_rf_detr(dependency, source, packaged)
|
||||
|
||||
|
||||
def _validate_component(
|
||||
module_id: str,
|
||||
result: dict[str, object],
|
||||
source: dict[str, object],
|
||||
packaged: dict[str, PortableResultArtifact],
|
||||
) -> None:
|
||||
frame_count = source.get("frame_count")
|
||||
camera_input = source.get("camera_input_sha256")
|
||||
if not isinstance(frame_count, int) or isinstance(frame_count, bool) or frame_count < 1:
|
||||
raise PortableResultPackageIntegrityError("AI-layer frame count is invalid")
|
||||
if module_id == "eomt":
|
||||
rows = result.get("artifacts")
|
||||
if not isinstance(rows, list):
|
||||
raise PortableResultPackageIntegrityError("EoMT artifacts are invalid")
|
||||
archive = next(
|
||||
(
|
||||
row
|
||||
for row in cast(list[object], rows)
|
||||
if isinstance(row, dict) and row.get("kind") == "panoptic-mask-archive"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if (
|
||||
result.get("schema_version") != "missioncore.recorded-perception-result/v2"
|
||||
or result.get("session_id") != source.get("session_id")
|
||||
or result.get("input_sha256") != camera_input
|
||||
or result.get("frames_processed") != frame_count
|
||||
or not isinstance(archive, dict)
|
||||
or archive.get("sha256") != packaged["eomt-panoptic-mask-archive"].sha256
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("EoMT result binding changed")
|
||||
return
|
||||
if module_id == "rf-detr":
|
||||
_validate_rf_detr(result, source, packaged)
|
||||
return
|
||||
if module_id == "object-distance":
|
||||
if (
|
||||
result.get("schema_version")
|
||||
!= "missioncore.observatory-ai-module-object-distance-result/v1"
|
||||
or result.get("module_id") != "object-distance"
|
||||
or result.get("source_session_id") != source.get("session_id")
|
||||
or result.get("frame_count") != frame_count
|
||||
or result.get("object_distances_sha256")
|
||||
!= packaged["object-distance-frame-observations"].sha256
|
||||
or result.get("range_estimator") != "median-camera-z-of-owned-current-points/v1"
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("object-distance result binding changed")
|
||||
return
|
||||
semantics = _object(result.get("video_semantics"), "DDRNet semantics")
|
||||
archive = _object(semantics.get("mask_archive"), "DDRNet mask archive")
|
||||
source_row = _object(result.get("source"), "DDRNet source")
|
||||
if (
|
||||
result.get("schema_version") != "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||
or source_row.get("input_count") != frame_count
|
||||
or archive.get("sha256") != packaged["ddrnet-semantic-mask-archive"].sha256
|
||||
or archive.get("frame_count") != frame_count
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("DDRNet result binding changed")
|
||||
|
||||
|
||||
def _validate_rf_detr(
|
||||
result: dict[str, object],
|
||||
source: dict[str, object],
|
||||
packaged: dict[str, PortableResultArtifact],
|
||||
) -> None:
|
||||
source_row = _object(result.get("source"), "RF-DETR source")
|
||||
if (
|
||||
result.get("schema_version") != "missioncore.observatory-ai-module-rf-detr-result/v1"
|
||||
or result.get("module_id") != "rf-detr"
|
||||
or source_row.get("session_id") != source.get("session_id")
|
||||
or result.get("frame_count") != source.get("frame_count")
|
||||
or result.get("detections_sha256") != packaged["rf-detr-frame-detections"].sha256
|
||||
):
|
||||
raise PortableResultPackageIntegrityError("RF-DETR result binding changed")
|
||||
|
||||
|
||||
def _artifact(value: object) -> PortableResultArtifact:
|
||||
row = _object(value, "AI-layer artifact")
|
||||
if set(row) != {"role", "relative_path", "media_type", "byte_length", "sha256"}:
|
||||
raise PortableResultPackageIntegrityError("AI-layer artifact fields changed")
|
||||
try:
|
||||
return PortableResultArtifact(**row) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise PortableResultPackageIntegrityError("AI-layer artifact is invalid") from exc
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, object]:
|
||||
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
||||
raise PortableResultPackageIntegrityError(f"{label} is invalid")
|
||||
return cast(dict[str, object], value)
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
@@ -52,12 +52,8 @@ from k1link.observatory.source_admission import (
|
||||
)
|
||||
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
|
||||
|
||||
PORTABLE_LAB_V1_SOURCE_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-lab-v1-source/v1"
|
||||
)
|
||||
PORTABLE_LAB_V1_PLAN_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-lab-v1-orchestration-plan/v1"
|
||||
)
|
||||
PORTABLE_LAB_V1_SOURCE_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-source/v1"
|
||||
PORTABLE_LAB_V1_PLAN_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-orchestration-plan/v1"
|
||||
PORTABLE_LAB_V1_PLAN_IDENTITY_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-lab-v1-orchestration-plan-identity/v1"
|
||||
)
|
||||
@@ -74,9 +70,7 @@ PORTABLE_LAB_V1_RELEASE_IDENTITY_SCHEMA: Final = (
|
||||
PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA: Final = (
|
||||
"missioncore.observatory-portable-lab-v1-executor-seal/v2"
|
||||
)
|
||||
PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: Final = (
|
||||
"missioncore.lab-v1-eomt-ddrnet-portable-profile/v2"
|
||||
)
|
||||
PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: Final = "missioncore.lab-v1-eomt-ddrnet-portable-profile/v2"
|
||||
PORTABLE_LAB_V1_DDRNET_EFFECTIVE_CONFIG_SCHEMA: Final = (
|
||||
"missioncore.lab-v1-goose-vegetation-benchmark/v1"
|
||||
)
|
||||
@@ -97,9 +91,7 @@ _DDRNET_CANDIDATE_KEY: Final = "ddrnet"
|
||||
_DDRNET_CHECKPOINT_SHA256: Final = (
|
||||
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
)
|
||||
_GOOSE_MAPPING_SHA256: Final = (
|
||||
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
|
||||
)
|
||||
_GOOSE_MAPPING_SHA256: Final = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
|
||||
_OBSERVATORY_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
|
||||
_CAMERA_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$")
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
@@ -214,9 +206,7 @@ class PortableLabV1SourceInput:
|
||||
"session_id": self.source_session_id,
|
||||
"catalog_sha256": self.source_catalog_sha256,
|
||||
"bundle_sha256": self.source_bundle_sha256,
|
||||
"capability_manifest_sha256": (
|
||||
self.source_capability_manifest_sha256
|
||||
),
|
||||
"capability_manifest_sha256": (self.source_capability_manifest_sha256),
|
||||
"adapter_sha256": self.source_adapter_sha256,
|
||||
},
|
||||
"camera_compute_job": {
|
||||
@@ -341,13 +331,8 @@ def materialize_lab_v1_source_input(
|
||||
)
|
||||
if hashlib.sha256(source_bundle_bytes).hexdigest() != job.source_bundle_sha256:
|
||||
raise PortableLabV1SourceError("source bundle digest differs from the sealed job")
|
||||
if (
|
||||
hashlib.sha256(capability_bytes).hexdigest()
|
||||
!= job.source_capability_manifest_sha256
|
||||
):
|
||||
raise PortableLabV1SourceError(
|
||||
"source capability digest differs from the sealed job"
|
||||
)
|
||||
if hashlib.sha256(capability_bytes).hexdigest() != job.source_capability_manifest_sha256:
|
||||
raise PortableLabV1SourceError("source capability digest differs from the sealed job")
|
||||
_validate_source_documents(
|
||||
source_bundle=source_bundle,
|
||||
capability=capability,
|
||||
@@ -484,9 +469,7 @@ class PortableLabV1ReleaseCandidate:
|
||||
_digest(self.executor_image_sha256, "release executor image sha256")
|
||||
ids = tuple(asset.asset_id for asset in self.assets)
|
||||
if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)):
|
||||
raise PortableLabV1ReleaseError(
|
||||
"release assets must be unique and canonically ordered"
|
||||
)
|
||||
raise PortableLabV1ReleaseError("release assets must be unique and canonically ordered")
|
||||
if not self.phases or len(self.phases) != len(set(self.phases)):
|
||||
raise PortableLabV1ReleaseError("release phases are invalid")
|
||||
for phase in self.phases:
|
||||
@@ -561,9 +544,7 @@ class PortableLabV1ReleaseCandidate:
|
||||
return cls(
|
||||
release_id=_string(document["release_id"], "release id"),
|
||||
setup_id=_string(document["setup_id"], "release setup id"),
|
||||
definition_id=_string(
|
||||
document["definition_id"], "release definition id"
|
||||
),
|
||||
definition_id=_string(document["definition_id"], "release definition id"),
|
||||
definition_version=_positive_int(
|
||||
document["definition_version"], "release definition version"
|
||||
),
|
||||
@@ -579,9 +560,7 @@ class PortableLabV1ReleaseCandidate:
|
||||
assets=assets,
|
||||
phases=phases,
|
||||
declared_blockers=blockers,
|
||||
candidate_sha256=_string(
|
||||
document["candidate_sha256"], "release candidate sha256"
|
||||
),
|
||||
candidate_sha256=_string(document["candidate_sha256"], "release candidate sha256"),
|
||||
repository_root=_real_directory(repository_root, "repository root"),
|
||||
)
|
||||
|
||||
@@ -606,14 +585,10 @@ class PortableLabV1ReleaseCandidate:
|
||||
definition.setup_id != self.setup_id
|
||||
or definition.definition_id != self.definition_id
|
||||
or definition.version != self.definition_version
|
||||
or definition.executable_contract_sha256
|
||||
!= self.definition_contract_sha256
|
||||
or definition.result_contract.contract_sha256
|
||||
!= self.result_contract_sha256
|
||||
or definition.executable_contract_sha256 != self.definition_contract_sha256
|
||||
or definition.result_contract.contract_sha256 != self.result_contract_sha256
|
||||
):
|
||||
raise PortableLabV1ReleaseError(
|
||||
"release candidate belongs to another RunDefinition"
|
||||
)
|
||||
raise PortableLabV1ReleaseError("release candidate belongs to another RunDefinition")
|
||||
|
||||
def inspect(
|
||||
self,
|
||||
@@ -660,9 +635,7 @@ class PortableLabV1ReleaseCandidate:
|
||||
or self.executor_image_sha256 is None
|
||||
or len(inspection.matched_assets) != len(self.assets)
|
||||
):
|
||||
raise PortableLabV1ReleaseError(
|
||||
"portable LAB V1 executor candidate is not sealable"
|
||||
)
|
||||
raise PortableLabV1ReleaseError("portable LAB V1 executor candidate is not sealable")
|
||||
identity = {
|
||||
"schema_version": PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA,
|
||||
"release_id": self.release_id,
|
||||
@@ -725,9 +698,8 @@ class PortableLabV1PlanPhase:
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_pattern(self.phase_id, _IDENTIFIER, "plan phase id")
|
||||
if (
|
||||
not self.component_sha256s
|
||||
or self.component_sha256s != tuple(sorted(self.component_sha256s))
|
||||
if not self.component_sha256s or self.component_sha256s != tuple(
|
||||
sorted(self.component_sha256s)
|
||||
):
|
||||
raise PortableLabV1PlanError("plan component identities are not canonical")
|
||||
for digest_value in self.component_sha256s:
|
||||
@@ -787,10 +759,8 @@ class PortableLabV1OrchestrationPlan:
|
||||
_digest(value, label)
|
||||
if (
|
||||
self.source_input.observatory_job_id != self.observatory_job_id
|
||||
or self.source_input.observatory_request_sha256
|
||||
!= self.observatory_request_sha256
|
||||
or self.source_input.observatory_identity_sha256
|
||||
!= self.observatory_identity_sha256
|
||||
or self.source_input.observatory_request_sha256 != self.observatory_request_sha256
|
||||
or self.source_input.observatory_identity_sha256 != self.observatory_identity_sha256
|
||||
):
|
||||
raise PortableLabV1PlanError("plan source belongs to another job")
|
||||
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
|
||||
@@ -804,10 +774,7 @@ class PortableLabV1OrchestrationPlan:
|
||||
"result-v2-assembly",
|
||||
):
|
||||
raise PortableLabV1PlanError("combined LAB V1 phase order changed")
|
||||
if (
|
||||
canonical_sha256(self.effective_ddrnet_config)
|
||||
!= self.effective_ddrnet_config_sha256
|
||||
):
|
||||
if canonical_sha256(self.effective_ddrnet_config) != self.effective_ddrnet_config_sha256:
|
||||
raise PortableLabV1PlanError("effective DDRNet config digest changed")
|
||||
if canonical_sha256(self.identity_document()) != self.plan_sha256:
|
||||
raise PortableLabV1PlanError("orchestration plan identity changed")
|
||||
@@ -830,8 +797,7 @@ class PortableLabV1OrchestrationPlan:
|
||||
raise PortableLabV1PlanError("release inspection belongs to another candidate")
|
||||
expected_asset_ids = tuple(asset.asset_id for asset in release.assets)
|
||||
if release_inspection.ready and (
|
||||
release_inspection.matched_assets != expected_asset_ids
|
||||
or release_inspection.blockers
|
||||
release_inspection.matched_assets != expected_asset_ids or release_inspection.blockers
|
||||
):
|
||||
raise PortableLabV1PlanError(
|
||||
"ready release inspection does not admit every exact asset"
|
||||
@@ -1147,13 +1113,8 @@ class PortableLabV1ResultAssembly:
|
||||
roles = tuple(item.role for item in self.artifacts)
|
||||
if roles != tuple(sorted(roles)) or len(roles) != len(set(roles)):
|
||||
raise PortableLabV1ResultError("assembled artifacts are not canonical")
|
||||
result_artifact = tuple(
|
||||
item for item in self.artifacts if item.role == "result-document"
|
||||
)
|
||||
if (
|
||||
len(result_artifact) != 1
|
||||
or result_artifact[0].sha256 != self.result_document_sha256
|
||||
):
|
||||
result_artifact = tuple(item for item in self.artifacts if item.role == "result-document")
|
||||
if len(result_artifact) != 1 or result_artifact[0].sha256 != self.result_document_sha256:
|
||||
raise PortableLabV1ResultError("assembled result document is not bound")
|
||||
|
||||
|
||||
@@ -1298,9 +1259,7 @@ def assemble_lab_v1_result_v2(
|
||||
"session_id": plan.source_input.source_session_id,
|
||||
"catalog_sha256": plan.source_input.source_catalog_sha256,
|
||||
"bundle_sha256": plan.source_input.source_bundle_sha256,
|
||||
"capability_manifest_sha256": (
|
||||
plan.source_input.source_capability_manifest_sha256
|
||||
),
|
||||
"capability_manifest_sha256": (plan.source_input.source_capability_manifest_sha256),
|
||||
"camera_input_sha256": plan.source_input.camera_input_sha256,
|
||||
"frame_count": plan.source_input.frame_count,
|
||||
"timeline_start_seconds": plan.source_input.timeline_start_seconds,
|
||||
@@ -1311,9 +1270,7 @@ def assemble_lab_v1_result_v2(
|
||||
"definition_id": definition.definition_id,
|
||||
"version": definition.version,
|
||||
"definition_sha256": definition.definition_sha256,
|
||||
"result_contract_sha256": (
|
||||
definition.result_contract.contract_sha256
|
||||
),
|
||||
"result_contract_sha256": (definition.result_contract.contract_sha256),
|
||||
"release_candidate_sha256": plan.release_candidate_sha256,
|
||||
"plan_sha256": plan.plan_sha256,
|
||||
},
|
||||
@@ -1343,9 +1300,7 @@ def assemble_lab_v1_result_v2(
|
||||
)
|
||||
final = parent / result_id
|
||||
if final.exists():
|
||||
raise PortableLabV1ResultError(
|
||||
"an assembled result with this identity already exists"
|
||||
)
|
||||
raise PortableLabV1ResultError("an assembled result with this identity already exists")
|
||||
_fsync_tree(staging)
|
||||
os.replace(staging, final)
|
||||
_fsync_directory(parent)
|
||||
@@ -1429,8 +1384,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
definition.setup_id != _EXPECTED_SETUP_ID
|
||||
or definition.definition_id != _EXPECTED_DEFINITION_ID
|
||||
or definition.result_contract.result_schema != PORTABLE_LAB_V1_RESULT_SCHEMA
|
||||
or definition.result_contract.contract_sha256
|
||||
!= _EXPECTED_RESULT_CONTRACT_SHA256
|
||||
or definition.result_contract.contract_sha256 != _EXPECTED_RESULT_CONTRACT_SHA256
|
||||
):
|
||||
raise PortableLabV1ResultError("LAB V1 validator received another definition")
|
||||
document = dict(context.result_document)
|
||||
@@ -1444,8 +1398,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
identity = _object(document["identity"], "portable LAB V1 identity")
|
||||
if (
|
||||
canonical_sha256(identity) != document["identity_sha256"]
|
||||
or document["result_id"]
|
||||
!= f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
|
||||
or document["result_id"] != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
|
||||
):
|
||||
raise PortableLabV1ResultError("portable LAB V1 result identity changed")
|
||||
source = _object(document["source"], "portable LAB V1 source")
|
||||
@@ -1484,8 +1437,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
source.get("session_id") != context.job.source_session_id
|
||||
or source.get("catalog_sha256") != context.job.source_catalog_sha256
|
||||
or source.get("bundle_sha256") != context.job.source_bundle_sha256
|
||||
or source.get("capability_manifest_sha256")
|
||||
!= context.job.source_capability_manifest_sha256
|
||||
or source.get("capability_manifest_sha256") != context.job.source_capability_manifest_sha256
|
||||
or run_definition.get("setup_id") != definition.setup_id
|
||||
or run_definition.get("definition_id") != definition.definition_id
|
||||
or run_definition.get("version") != definition.version
|
||||
@@ -1499,13 +1451,10 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
raise PortableLabV1ResultError("portable LAB V1 artifacts are not an array")
|
||||
declared = tuple(_artifact_from_document(row) for row in artifact_rows)
|
||||
declared_roles = tuple(artifact.role for artifact in declared)
|
||||
if (
|
||||
declared_roles != tuple(sorted(declared_roles))
|
||||
or len(declared_roles) != len(set(declared_roles))
|
||||
if declared_roles != tuple(sorted(declared_roles)) or len(declared_roles) != len(
|
||||
set(declared_roles)
|
||||
):
|
||||
raise PortableLabV1ResultError(
|
||||
"portable result artifacts are not canonical"
|
||||
)
|
||||
raise PortableLabV1ResultError("portable result artifacts are not canonical")
|
||||
declared_by_role = {artifact.role: artifact for artifact in declared}
|
||||
package_by_role = {artifact.role: artifact for artifact in context.manifest.artifacts}
|
||||
if set(package_by_role) != {*declared_by_role, "result-document"}:
|
||||
@@ -1540,15 +1489,13 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
|
||||
if (
|
||||
source_input.identity_sha256 != identity.get("source_input_sha256")
|
||||
or plan.plan_sha256 != run_definition.get("plan_sha256")
|
||||
or plan.release_candidate_sha256
|
||||
!= run_definition.get("release_candidate_sha256")
|
||||
or plan.release_candidate_sha256 != run_definition.get("release_candidate_sha256")
|
||||
or plan.observatory_job_id != context.job.job_id
|
||||
or plan.observatory_request_sha256 != context.job.request_sha256
|
||||
or plan.observatory_identity_sha256 != context.job.identity_sha256
|
||||
or source.get("camera_input_sha256") != source_input.camera_input_sha256
|
||||
or source.get("frame_count") != source_input.frame_count
|
||||
or source.get("timeline_start_seconds")
|
||||
!= source_input.timeline_start_seconds
|
||||
or source.get("timeline_start_seconds") != source_input.timeline_start_seconds
|
||||
or source.get("timeline_end_seconds") != source_input.timeline_end_seconds
|
||||
):
|
||||
raise PortableLabV1ResultError("portable source or plan artifact changed")
|
||||
@@ -1621,9 +1568,7 @@ def portable_lab_v1_source_input_from_document(
|
||||
"portable camera compute job",
|
||||
)
|
||||
return PortableLabV1SourceInput(
|
||||
observatory_job_id=_string(
|
||||
observatory_job["job_id"], "portable source Observatory job id"
|
||||
),
|
||||
observatory_job_id=_string(observatory_job["job_id"], "portable source Observatory job id"),
|
||||
observatory_request_sha256=_string(
|
||||
observatory_job["request_sha256"],
|
||||
"portable source Observatory request sha256",
|
||||
@@ -1633,23 +1578,15 @@ def portable_lab_v1_source_input_from_document(
|
||||
"portable source Observatory identity sha256",
|
||||
),
|
||||
source_session_id=_string(source["session_id"], "portable source session id"),
|
||||
source_catalog_sha256=_string(
|
||||
source["catalog_sha256"], "portable source catalog sha256"
|
||||
),
|
||||
source_bundle_sha256=_string(
|
||||
source["bundle_sha256"], "portable source bundle sha256"
|
||||
),
|
||||
source_catalog_sha256=_string(source["catalog_sha256"], "portable source catalog sha256"),
|
||||
source_bundle_sha256=_string(source["bundle_sha256"], "portable source bundle sha256"),
|
||||
source_capability_manifest_sha256=_string(
|
||||
source["capability_manifest_sha256"],
|
||||
"portable source capability sha256",
|
||||
),
|
||||
source_adapter_sha256=_string(
|
||||
source["adapter_sha256"], "portable source adapter sha256"
|
||||
),
|
||||
source_adapter_sha256=_string(source["adapter_sha256"], "portable source adapter sha256"),
|
||||
camera_job_id=_string(camera["job_id"], "portable camera job id"),
|
||||
camera_input_sha256=_string(
|
||||
camera["input_sha256"], "portable camera input sha256"
|
||||
),
|
||||
camera_input_sha256=_string(camera["input_sha256"], "portable camera input sha256"),
|
||||
camera_source_id=_string(camera["source_id"], "portable camera source id"),
|
||||
codec_epoch=_positive_int(camera["codec_epoch"], "portable codec epoch"),
|
||||
input_byte_length=_positive_int(
|
||||
@@ -1659,15 +1596,11 @@ def portable_lab_v1_source_input_from_document(
|
||||
timeline_start_seconds=_finite_float(
|
||||
camera["timeline_start_seconds"], "portable timeline start"
|
||||
),
|
||||
timeline_end_seconds=_finite_float(
|
||||
camera["timeline_end_seconds"], "portable timeline end"
|
||||
),
|
||||
timeline_end_seconds=_finite_float(camera["timeline_end_seconds"], "portable timeline end"),
|
||||
camera_generation_sha256=_string(
|
||||
camera["generation_sha256"], "portable camera generation sha256"
|
||||
),
|
||||
calibration_sha256=_string(
|
||||
camera["calibration_sha256"], "portable calibration sha256"
|
||||
),
|
||||
calibration_sha256=_string(camera["calibration_sha256"], "portable calibration sha256"),
|
||||
)
|
||||
|
||||
|
||||
@@ -1758,9 +1691,7 @@ def portable_lab_v1_orchestration_plan_from_document(
|
||||
raise PortableLabV1ResultError("portable plan admission changed")
|
||||
phases = tuple(_plan_phase_from_document(value) for value in phases_value)
|
||||
plan = PortableLabV1OrchestrationPlan(
|
||||
observatory_job_id=_string(
|
||||
observatory_job["job_id"], "portable plan Observatory job id"
|
||||
),
|
||||
observatory_job_id=_string(observatory_job["job_id"], "portable plan Observatory job id"),
|
||||
observatory_request_sha256=_string(
|
||||
observatory_job["request_sha256"],
|
||||
"portable plan Observatory request sha256",
|
||||
@@ -1770,9 +1701,7 @@ def portable_lab_v1_orchestration_plan_from_document(
|
||||
"portable plan Observatory identity sha256",
|
||||
),
|
||||
setup_id=_string(run_definition["setup_id"], "portable plan setup id"),
|
||||
definition_id=_string(
|
||||
run_definition["definition_id"], "portable plan definition id"
|
||||
),
|
||||
definition_id=_string(run_definition["definition_id"], "portable plan definition id"),
|
||||
definition_version=_positive_int(
|
||||
run_definition["version"], "portable plan definition version"
|
||||
),
|
||||
@@ -1817,8 +1746,7 @@ def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase:
|
||||
input_values = row["input_roles"]
|
||||
output_values = row["output_roles"]
|
||||
if not all(
|
||||
isinstance(values, list)
|
||||
for values in (component_values, input_values, output_values)
|
||||
isinstance(values, list) for values in (component_values, input_values, output_values)
|
||||
):
|
||||
raise PortableLabV1ResultError("portable plan phase arrays changed")
|
||||
return PortableLabV1PlanPhase(
|
||||
@@ -1828,12 +1756,10 @@ def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase:
|
||||
for item in cast(list[object], component_values)
|
||||
),
|
||||
input_roles=tuple(
|
||||
_string(item, "portable plan input role")
|
||||
for item in cast(list[object], input_values)
|
||||
_string(item, "portable plan input role") for item in cast(list[object], input_values)
|
||||
),
|
||||
output_roles=tuple(
|
||||
_string(item, "portable plan output role")
|
||||
for item in cast(list[object], output_values)
|
||||
_string(item, "portable plan output role") for item in cast(list[object], output_values)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1859,8 +1785,7 @@ def _validate_source_documents(
|
||||
source_bundle.get("source_session_id") != job.source_session_id
|
||||
or capability.get("source_session_id") != job.source_session_id
|
||||
or camera_job.session_id != job.source_session_id
|
||||
or source_bundle.get("source_catalog_sha256")
|
||||
!= job.source_catalog_sha256
|
||||
or source_bundle.get("source_catalog_sha256") != job.source_catalog_sha256
|
||||
or capability.get("source_catalog_sha256") != job.source_catalog_sha256
|
||||
or capability.get("source_bundle_sha256") != job.source_bundle_sha256
|
||||
or capability.get("source_adapter_sha256") != job.source_adapter_sha256
|
||||
@@ -1897,8 +1822,7 @@ def _validate_source_documents(
|
||||
raise PortableLabV1SourceError("source camera segment count changed")
|
||||
if (
|
||||
video.get("source_id") != requirements.camera_source_id
|
||||
or video.get("semantic_channel_id")
|
||||
!= requirements.camera_semantic_channel_id
|
||||
or video.get("semantic_channel_id") != requirements.camera_semantic_channel_id
|
||||
or video.get("seekable") is not True
|
||||
or camera_job.source_id != camera.get("public_source_id")
|
||||
or camera_job.codec_epoch != epoch.get("ordinal")
|
||||
@@ -1934,18 +1858,19 @@ def _validate_source_documents(
|
||||
if not isinstance(files, list):
|
||||
raise PortableLabV1SourceError("camera compute job file set is invalid")
|
||||
file_rows = {
|
||||
PurePosixPath(_string(_object(row, "camera file").get("path"), "camera file path")).name:
|
||||
_object(row, "camera file")
|
||||
PurePosixPath(
|
||||
_string(_object(row, "camera file").get("path"), "camera file path")
|
||||
).name: _object(row, "camera file")
|
||||
for row in files
|
||||
if PurePosixPath(
|
||||
_string(_object(row, "camera file").get("path"), "camera file path")
|
||||
).parent.name
|
||||
in (f"epoch-{camera_job.codec_epoch}", "segments")
|
||||
}
|
||||
if (
|
||||
init.get("sha256") != _object(file_rows.get("init.mp4"), "camera init file").get("sha256")
|
||||
or init.get("byte_length")
|
||||
!= _object(file_rows.get("init.mp4"), "camera init file").get("byte_length")
|
||||
if init.get("sha256") != _object(file_rows.get("init.mp4"), "camera init file").get(
|
||||
"sha256"
|
||||
) or init.get("byte_length") != _object(file_rows.get("init.mp4"), "camera init file").get(
|
||||
"byte_length"
|
||||
):
|
||||
raise PortableLabV1SourceError("camera init differs from the admitted source")
|
||||
for index, row in enumerate(segments, start=1):
|
||||
@@ -1964,11 +1889,7 @@ def _verify_definition_and_job(
|
||||
job: SealedObservatoryRecordedJob,
|
||||
) -> None:
|
||||
if (
|
||||
definition.setup_id != _EXPECTED_SETUP_ID
|
||||
or definition.definition_id != _EXPECTED_DEFINITION_ID
|
||||
or definition.result_contract.contract_sha256
|
||||
!= _EXPECTED_RESULT_CONTRACT_SHA256
|
||||
or job.setup_id != definition.setup_id
|
||||
job.setup_id != definition.setup_id
|
||||
or job.definition_id != definition.definition_id
|
||||
or job.definition_version != definition.version
|
||||
or job.definition_sha256 != definition.definition_sha256
|
||||
@@ -1977,14 +1898,14 @@ def _verify_definition_and_job(
|
||||
or job.source_adapter_sha256 != definition.source_adapter.contract_sha256
|
||||
or job.model_release_ids != definition.learned_models
|
||||
or job.resource_profile_id != definition.resource_profile.profile_id
|
||||
or job.executor_identity.model_manifest_sha256
|
||||
!= definition.model_manifest_sha256
|
||||
or job.executor_identity.model_manifest_sha256 != definition.model_manifest_sha256
|
||||
or job.executor_identity.resource_profile_sha256
|
||||
!= definition.resource_profile.profile_sha256
|
||||
or job.checkpoint_policy != definition.resource_profile.checkpoint_policy
|
||||
or job.allowed_checkpoints != definition.resource_profile.allowed_checkpoints
|
||||
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise PortableLabV1SourceError("job is not the exact portable LAB V1 definition")
|
||||
raise PortableLabV1SourceError("job is not the exact portable definition")
|
||||
|
||||
|
||||
def _verify_source_and_job(
|
||||
@@ -2008,8 +1929,7 @@ def _verify_source_and_job(
|
||||
or source.source_session_id != job.source_session_id
|
||||
or source.source_catalog_sha256 != job.source_catalog_sha256
|
||||
or source.source_bundle_sha256 != job.source_bundle_sha256
|
||||
or source.source_capability_manifest_sha256
|
||||
!= job.source_capability_manifest_sha256
|
||||
or source.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
|
||||
or source.source_adapter_sha256 != job.source_adapter_sha256
|
||||
or source.calibration_sha256 != requirements.calibration_identity_sha256
|
||||
):
|
||||
@@ -2069,8 +1989,7 @@ def _verify_plan_definition(
|
||||
or plan.definition_id != definition.definition_id
|
||||
or plan.definition_version != definition.version
|
||||
or plan.definition_sha256 != definition.definition_sha256
|
||||
or plan.result_contract_sha256
|
||||
!= definition.result_contract.contract_sha256
|
||||
or plan.result_contract_sha256 != definition.result_contract.contract_sha256
|
||||
):
|
||||
raise PortableLabV1PlanError("orchestration plan belongs to another definition")
|
||||
|
||||
@@ -2092,8 +2011,7 @@ def _validate_eomt_component(
|
||||
or document.get("source_id") != plan.source_input.camera_source_id
|
||||
or document.get("codec_epoch") != plan.source_input.codec_epoch
|
||||
or document.get("timestamp_basis") != "session-time-seconds"
|
||||
or document.get("timeline_start_seconds")
|
||||
!= plan.source_input.timeline_start_seconds
|
||||
or document.get("timeline_start_seconds") != plan.source_input.timeline_start_seconds
|
||||
or document.get("timeline_end_seconds") != plan.source_input.timeline_end_seconds
|
||||
or document.get("frames_processed") != plan.source_input.frame_count
|
||||
or document.get("ground_truth") is not False
|
||||
@@ -2186,10 +2104,8 @@ def _validate_ddrnet_component(
|
||||
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
|
||||
or semantics.get("outside_crop_state") != "undefined"
|
||||
or semantics.get("base_m4_result_id") is not None
|
||||
or provenance.get("config_sha256")
|
||||
!= plan.effective_ddrnet_config_sha256
|
||||
or provenance.get("policy_sha256")
|
||||
!= components["vegetation-mission-policy-v1"].sha256
|
||||
or provenance.get("config_sha256") != plan.effective_ddrnet_config_sha256
|
||||
or provenance.get("policy_sha256") != components["vegetation-mission-policy-v1"].sha256
|
||||
or provenance.get("provider_map_sha256")
|
||||
!= components["vegetation-provider-label-map-v1"].sha256
|
||||
or authority
|
||||
@@ -2290,8 +2206,7 @@ def _validate_assembly(
|
||||
if (
|
||||
canonical_sha256(identity) != document["identity_sha256"]
|
||||
or document["result_id"] != assembly.result_id
|
||||
or document["result_id"]
|
||||
!= f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
|
||||
or document["result_id"] != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
|
||||
):
|
||||
raise PortableLabV1ResultError("assembled result identity changed")
|
||||
for artifact in assembly.artifacts:
|
||||
@@ -2450,10 +2365,8 @@ def _validate_published_eomt_component(
|
||||
or document.get("source_id") != plan.source_input.camera_source_id
|
||||
or document.get("codec_epoch") != plan.source_input.codec_epoch
|
||||
or document.get("timestamp_basis") != "session-time-seconds"
|
||||
or document.get("timeline_start_seconds")
|
||||
!= plan.source_input.timeline_start_seconds
|
||||
or document.get("timeline_end_seconds")
|
||||
!= plan.source_input.timeline_end_seconds
|
||||
or document.get("timeline_start_seconds") != plan.source_input.timeline_start_seconds
|
||||
or document.get("timeline_end_seconds") != plan.source_input.timeline_end_seconds
|
||||
or document.get("frames_processed") != plan.source_input.frame_count
|
||||
or document.get("ground_truth") is not False
|
||||
or semantic.get("id") != model.model_id
|
||||
@@ -2472,9 +2385,7 @@ def _validate_published_eomt_component(
|
||||
}
|
||||
if not isinstance(rows, list) or len(rows) != len(expected_kinds):
|
||||
raise PortableLabV1ResultError("published EoMT artifact set changed")
|
||||
package_by_role = {
|
||||
artifact.role: artifact for artifact in context.manifest.artifacts
|
||||
}
|
||||
package_by_role = {artifact.role: artifact for artifact in context.manifest.artifacts}
|
||||
observed_kinds: set[str] = set()
|
||||
for value in rows:
|
||||
row = _object(value, "published EoMT artifact")
|
||||
@@ -2482,13 +2393,9 @@ def _validate_published_eomt_component(
|
||||
if kind not in expected_kinds or kind in observed_kinds:
|
||||
raise PortableLabV1ResultError("published EoMT artifact roles changed")
|
||||
observed_kinds.add(kind)
|
||||
relative = _safe_relative_path(
|
||||
_string(row.get("path"), "published EoMT artifact path")
|
||||
)
|
||||
relative = _safe_relative_path(_string(row.get("path"), "published EoMT artifact path"))
|
||||
if len(relative.parts) != 1:
|
||||
raise PortableLabV1ResultError(
|
||||
"published EoMT artifact path changed"
|
||||
)
|
||||
raise PortableLabV1ResultError("published EoMT artifact path changed")
|
||||
packaged = package_by_role.get(f"eomt-{kind}")
|
||||
if (
|
||||
packaged is None
|
||||
@@ -2496,9 +2403,7 @@ def _validate_published_eomt_component(
|
||||
or row.get("byte_length") != packaged.byte_length
|
||||
or row.get("sha256") != packaged.sha256
|
||||
):
|
||||
raise PortableLabV1ResultError(
|
||||
"published EoMT artifact identity changed"
|
||||
)
|
||||
raise PortableLabV1ResultError("published EoMT artifact identity changed")
|
||||
|
||||
|
||||
def _validate_published_ddrnet_component(
|
||||
@@ -2557,10 +2462,8 @@ def _validate_published_ddrnet_component(
|
||||
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
|
||||
or semantics.get("outside_crop_state") != "undefined"
|
||||
or semantics.get("base_m4_result_id") is not None
|
||||
or provenance.get("config_sha256")
|
||||
!= plan.effective_ddrnet_config_sha256
|
||||
or provenance.get("policy_sha256")
|
||||
!= components["vegetation-mission-policy-v1"].sha256
|
||||
or provenance.get("config_sha256") != plan.effective_ddrnet_config_sha256
|
||||
or provenance.get("policy_sha256") != components["vegetation-mission-policy-v1"].sha256
|
||||
or provenance.get("provider_map_sha256")
|
||||
!= components["vegetation-provider-label-map-v1"].sha256
|
||||
or package_archive is None
|
||||
@@ -2595,6 +2498,8 @@ def _validate_published_ddrnet_component(
|
||||
"lab-v1-ravnoves-video-ddrnet-" + canonical_sha256(identity_value)
|
||||
):
|
||||
raise PortableLabV1ResultError("published DDRNet result identity changed")
|
||||
|
||||
|
||||
def _model(definition: PortableRunDefinition, release_id: str): # type: ignore[no-untyped-def]
|
||||
for model in definition.models:
|
||||
if model.release_id == release_id:
|
||||
@@ -2644,9 +2549,8 @@ def _release_binding_matches(
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return False
|
||||
return (
|
||||
(asset.byte_length is None or path.stat().st_size == asset.byte_length)
|
||||
and _sha256_file(path) == asset.sha256
|
||||
)
|
||||
asset.byte_length is None or path.stat().st_size == asset.byte_length
|
||||
) and _sha256_file(path) == asset.sha256
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
@@ -94,9 +94,7 @@ _EXPECTED_RESULT_CONTRACT_SHA256: Final = (
|
||||
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
|
||||
)
|
||||
_PORTABLE_CONFIG_ASSET_ID: Final = "ddrnet-portable-config"
|
||||
_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = (
|
||||
"lab-v1-worker-installation-receipt"
|
||||
)
|
||||
_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = "lab-v1-worker-installation-receipt"
|
||||
_MAX_SOURCE_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
|
||||
_MAX_MATERIALIZATION_MANIFEST_BYTES: Final = 64 * 1024 * 1024
|
||||
_MAX_SOURCE_MEMBERS: Final = 100_000
|
||||
@@ -432,6 +430,27 @@ def materialize_lab_v1_source_from_worker_stage(
|
||||
"""Build a deterministic camera job from only manifested Worker members."""
|
||||
|
||||
_verify_definition(definition)
|
||||
return materialize_recorded_camera_source_from_worker_stage(
|
||||
worker_stage=worker_stage,
|
||||
job=job,
|
||||
definition=definition,
|
||||
output_parent=output_parent,
|
||||
)
|
||||
|
||||
|
||||
def materialize_recorded_camera_source_from_worker_stage(
|
||||
*,
|
||||
worker_stage: PortableWorkerSourceStage,
|
||||
job: SealedObservatoryRecordedJob,
|
||||
definition: PortableRunDefinition,
|
||||
output_parent: Path,
|
||||
) -> PortableLabV1MaterializedSource:
|
||||
"""Build the shared recorded-camera input for any exact installed definition.
|
||||
|
||||
The historical LAB entrypoint above retains its setup-specific admission.
|
||||
New modular packages use this model-neutral source preparation boundary.
|
||||
"""
|
||||
|
||||
if (
|
||||
worker_stage.source_bundle_sha256 != job.source_bundle_sha256
|
||||
or worker_stage.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
|
||||
@@ -574,8 +593,7 @@ def _verify_candidate_release(
|
||||
or candidate.definition_id != release.definition_id
|
||||
or candidate.definition_version != release.definition_version
|
||||
or candidate.definition_sha256 != definition.definition_sha256
|
||||
or release.definition_contract_sha256
|
||||
!= definition.executable_contract_sha256
|
||||
or release.definition_contract_sha256 != definition.executable_contract_sha256
|
||||
or candidate.result_contract_sha256 != release.result_contract_sha256
|
||||
or tuple(phase.phase_id for phase in candidate.phases) != PORTABLE_LAB_V1_RUNTIME_PHASES
|
||||
or executor is None
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Project sealed RF-DETR boxes and optional K1 ranges without inference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import rerun as rr
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.observatory.portable_tgs_replay import PortableReplayError, read_json, verified_file
|
||||
from k1link.perception.contracts import ObjectProposal2D, ObstacleObservation
|
||||
|
||||
RESULT_SCHEMA = "missioncore.recorded-ai-layer-review/v1"
|
||||
RF_RESULT_SCHEMA = "missioncore.observatory-ai-module-rf-detr-result/v1"
|
||||
RANGE_RESULT_SCHEMA = "missioncore.observatory-ai-module-object-distance-result/v1"
|
||||
RF_ROW_SCHEMA = "missioncore.observatory-ai-module-rf-detr-frame/v1"
|
||||
RANGE_ROW_SCHEMA = "missioncore.observatory-ai-module-object-distance-frame/v1"
|
||||
RENDERER_VERSION = "portable-rf-detr-k1-range-rerun-0.36.3-v1"
|
||||
_MAX_FRAMES = 250_000
|
||||
_MAX_ROW_BYTES = 8 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectReplayFrame:
|
||||
session_seconds: float
|
||||
proposals: tuple[ObjectProposal2D, ...]
|
||||
ranges_m: dict[str, float | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectReplayData:
|
||||
frames: tuple[ObjectReplayFrame, ...]
|
||||
end_seconds: float
|
||||
include_ranges: bool
|
||||
|
||||
|
||||
def _artifact(
|
||||
members: dict[str, dict[str, Any]],
|
||||
store: CentralArtifactStore,
|
||||
role: str,
|
||||
media_type: str,
|
||||
) -> Path:
|
||||
member = members.get(role)
|
||||
if member is None or member.get("media_type") != media_type:
|
||||
raise PortableReplayError("object replay artifact is missing")
|
||||
return verified_file(
|
||||
store.object_path(member["sha256"]), member["sha256"], member["byte_length"]
|
||||
)
|
||||
|
||||
|
||||
def _rows(path: Path, *, schema: str, count: int) -> list[dict[str, Any]]:
|
||||
result: list[dict[str, Any]] = []
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
for raw in stream:
|
||||
if len(raw) > _MAX_ROW_BYTES or len(result) >= count:
|
||||
raise PortableReplayError("object replay rows exceed their bound")
|
||||
try:
|
||||
row = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise PortableReplayError("object replay row is invalid") from exc
|
||||
if not isinstance(row, dict) or row.get("schema_version") != schema:
|
||||
raise PortableReplayError("object replay row contract changed")
|
||||
result.append(row)
|
||||
if len(result) != count:
|
||||
raise PortableReplayError("object replay frame count changed")
|
||||
return result
|
||||
|
||||
|
||||
def _time(value: object) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise PortableReplayError("object replay time is invalid")
|
||||
number = float(value)
|
||||
if not math.isfinite(number):
|
||||
raise PortableReplayError("object replay time is invalid")
|
||||
return number
|
||||
|
||||
|
||||
def load_object_data(
|
||||
view: dict[str, Any],
|
||||
store: CentralArtifactStore,
|
||||
*,
|
||||
source_bundle_sha256: str,
|
||||
starts: list[float],
|
||||
end_seconds: float,
|
||||
) -> ObjectReplayData:
|
||||
doc = view["result_document"]
|
||||
source = doc.get("source")
|
||||
module = doc.get("module")
|
||||
if not isinstance(source, dict) or not isinstance(module, dict):
|
||||
raise PortableReplayError("object replay result is malformed")
|
||||
module_id = module.get("module_id")
|
||||
if (
|
||||
doc.get("schema_version") != RESULT_SCHEMA
|
||||
or doc.get("result_id") != view["result_id"]
|
||||
or module_id not in {"rf-detr", "object-distance"}
|
||||
or source.get("session_id") != view["source_session_id"]
|
||||
or source.get("bundle_sha256") != source_bundle_sha256
|
||||
or not 0 < len(starts) <= _MAX_FRAMES
|
||||
or source.get("frame_count") != len(starts)
|
||||
or source.get("timeline_start_seconds") != starts[0]
|
||||
or source.get("timeline_end_seconds") != end_seconds
|
||||
or not np.isfinite([*starts, end_seconds]).all()
|
||||
or np.any(np.diff([*starts, end_seconds]) <= 0)
|
||||
):
|
||||
raise PortableReplayError("object result and source clock disagree")
|
||||
members = {row["role"]: row for row in view["artifacts"]}
|
||||
if len(members) != len(view["artifacts"]):
|
||||
raise PortableReplayError("object artifact roles are duplicated")
|
||||
rf_path = _artifact(members, store, "rf-detr-result-document", "application/json")
|
||||
rf = read_json(rf_path)
|
||||
detection_path = _artifact(members, store, "rf-detr-frame-detections", "application/x-ndjson")
|
||||
if (
|
||||
rf.get("schema_version") != RF_RESULT_SCHEMA
|
||||
or rf.get("module_id") != "rf-detr"
|
||||
or not isinstance(rf.get("source"), dict)
|
||||
or rf["source"].get("session_id") != view["source_session_id"]
|
||||
or rf.get("frame_count") != len(starts)
|
||||
or rf.get("detections_sha256") != members["rf-detr-frame-detections"]["sha256"]
|
||||
or (
|
||||
module_id == "rf-detr"
|
||||
and module.get("component_result_sha256")
|
||||
!= members["rf-detr-result-document"]["sha256"]
|
||||
)
|
||||
):
|
||||
raise PortableReplayError("RF-DETR result binding changed")
|
||||
detection_rows = _rows(detection_path, schema=RF_ROW_SCHEMA, count=len(starts))
|
||||
|
||||
range_rows: list[dict[str, Any]] | None = None
|
||||
if module_id == "object-distance":
|
||||
range_path = _artifact(
|
||||
members,
|
||||
store,
|
||||
"object-distance-frame-observations",
|
||||
"application/x-ndjson",
|
||||
)
|
||||
ranged = read_json(
|
||||
_artifact(members, store, "object-distance-result-document", "application/json")
|
||||
)
|
||||
if (
|
||||
ranged.get("schema_version") != RANGE_RESULT_SCHEMA
|
||||
or ranged.get("module_id") != "object-distance"
|
||||
or ranged.get("source_session_id") != view["source_session_id"]
|
||||
or ranged.get("frame_count") != len(starts)
|
||||
or ranged.get("object_distances_sha256")
|
||||
!= members["object-distance-frame-observations"]["sha256"]
|
||||
or module.get("component_result_sha256")
|
||||
!= members["object-distance-result-document"]["sha256"]
|
||||
):
|
||||
raise PortableReplayError("object-distance result binding changed")
|
||||
range_rows = _rows(range_path, schema=RANGE_ROW_SCHEMA, count=len(starts))
|
||||
|
||||
frames: list[ObjectReplayFrame] = []
|
||||
try:
|
||||
for index, (timestamp, row) in enumerate(zip(starts, detection_rows, strict=True)):
|
||||
if row.get("frame_index") != index or _time(row.get("session_seconds")) != timestamp:
|
||||
raise PortableReplayError("RF-DETR frame clock changed")
|
||||
raw_proposals = row.get("proposals")
|
||||
if not isinstance(raw_proposals, list):
|
||||
raise PortableReplayError("RF-DETR proposals are unavailable")
|
||||
proposals = tuple(ObjectProposal2D.from_dict(value) for value in raw_proposals)
|
||||
proposal_ids = {item.proposal_id for item in proposals}
|
||||
if len(proposal_ids) != len(proposals) or any(
|
||||
item.region.x_max > 800 or item.region.y_max > 600 for item in proposals
|
||||
):
|
||||
raise PortableReplayError("RF-DETR proposal geometry changed")
|
||||
ranges: dict[str, float | None] = {}
|
||||
if range_rows is not None:
|
||||
range_row = range_rows[index]
|
||||
if (
|
||||
range_row.get("frame_index") != index
|
||||
or _time(range_row.get("session_seconds")) != timestamp
|
||||
or not isinstance(range_row.get("observations"), list)
|
||||
):
|
||||
raise PortableReplayError("object-distance frame clock changed")
|
||||
for raw in range_row["observations"]:
|
||||
observation = ObstacleObservation.from_dict(raw)
|
||||
distance = (
|
||||
None
|
||||
if observation.metric_geometry is None
|
||||
else observation.metric_geometry.range_m
|
||||
)
|
||||
for proposal_id in observation.proposal_ids:
|
||||
if proposal_id not in proposal_ids or proposal_id in ranges:
|
||||
raise PortableReplayError("object-distance proposal binding changed")
|
||||
ranges[proposal_id] = distance
|
||||
if set(ranges) != proposal_ids:
|
||||
raise PortableReplayError("object-distance coverage changed")
|
||||
frames.append(ObjectReplayFrame(timestamp, proposals, ranges))
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
if isinstance(exc, PortableReplayError):
|
||||
raise
|
||||
raise PortableReplayError("object replay contract changed") from exc
|
||||
return ObjectReplayData(tuple(frames), end_seconds, range_rows is not None)
|
||||
|
||||
|
||||
def _color(label: str) -> list[int]:
|
||||
digest = hashlib.sha256(f"mission-core-object-{label}".encode()).digest()
|
||||
return [80 + digest[channel] % 160 for channel in range(3)] + [255]
|
||||
|
||||
|
||||
def log_objects(recording: rr.RecordingStream, data: ObjectReplayData) -> None:
|
||||
for frame in data.frames:
|
||||
recording.set_time(
|
||||
"session_time",
|
||||
duration=np.timedelta64(round(frame.session_seconds * 1e9), "ns"),
|
||||
)
|
||||
if not frame.proposals:
|
||||
recording.log("/perception/camera/detections", rr.Clear(recursive=False))
|
||||
continue
|
||||
labels: list[str] = []
|
||||
for proposal in frame.proposals:
|
||||
label = proposal.semantic_hint or "объект"
|
||||
confidence = f"{proposal.objectness * 100:.0f}%"
|
||||
if data.include_ranges:
|
||||
distance = frame.ranges_m[proposal.proposal_id]
|
||||
suffix = "дальность н/д" if distance is None else f"{distance:.1f} м"
|
||||
labels.append(f"{label} · {confidence} · {suffix}")
|
||||
else:
|
||||
labels.append(f"{label} · {confidence}")
|
||||
recording.log(
|
||||
"/perception/camera/detections",
|
||||
rr.Boxes2D(
|
||||
array=[proposal.region.as_tuple() for proposal in frame.proposals],
|
||||
array_format=rr.Box2DFormat.XYXY,
|
||||
labels=labels,
|
||||
colors=[_color(proposal.semantic_hint or "object") for proposal in frame.proposals],
|
||||
show_labels=True,
|
||||
),
|
||||
)
|
||||
recording.set_time("session_time", duration=np.timedelta64(round(data.end_seconds * 1e9), "ns"))
|
||||
recording.log("/perception/camera/detections", rr.Clear(recursive=False))
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@@ -14,6 +14,7 @@ from .portable_artifact_transport import (
|
||||
from .portable_result_contract import PortableResultPublisherError
|
||||
from .portable_result_publisher import PortableObservatoryResultPublisher
|
||||
from .recorded_jobs import (
|
||||
ObservatoryRecordedJob,
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
@@ -55,20 +56,19 @@ class PortablePublicationReconciler:
|
||||
now = self.clock()
|
||||
if now.tzinfo is None:
|
||||
raise ValueError("publication reconciliation clock must be timezone-aware")
|
||||
candidates = self.queue.pending_publications()[:limit]
|
||||
examined = 0
|
||||
published = 0
|
||||
failed = 0
|
||||
deferred = 0
|
||||
exhausted = 0
|
||||
for job in candidates:
|
||||
for job in self._candidates():
|
||||
examined += 1
|
||||
attempts = job.publication_attempts
|
||||
if attempts >= self.maximum_attempts:
|
||||
exhausted += 1
|
||||
continue
|
||||
updated_at = _timestamp(job.updated_at_utc)
|
||||
retry_at = updated_at + timedelta(
|
||||
seconds=self.retry_delays_seconds[attempts]
|
||||
)
|
||||
retry_at = updated_at + timedelta(seconds=self.retry_delays_seconds[attempts])
|
||||
if now < retry_at:
|
||||
deferred += 1
|
||||
continue
|
||||
@@ -82,24 +82,37 @@ class PortablePublicationReconciler:
|
||||
with suppress(ObservatoryRecordedQueueError):
|
||||
self.queue.mark_publication_failed(job.job_id, message=message)
|
||||
failed += 1
|
||||
# Bound actual publication work, not the first rows of the outbox.
|
||||
# Exhausted/backoff entries remain evidence but cannot indefinitely
|
||||
# hide later ready results behind the same prefix on every tick.
|
||||
if published + failed >= limit:
|
||||
break
|
||||
return PortablePublicationReconciliation(
|
||||
examined=len(candidates),
|
||||
examined=examined,
|
||||
published=published,
|
||||
failed=failed,
|
||||
deferred=deferred,
|
||||
exhausted=exhausted,
|
||||
)
|
||||
|
||||
def _candidates(self) -> Iterator[ObservatoryRecordedJob]:
|
||||
after: tuple[str, str] | None = None
|
||||
while True:
|
||||
page = self.queue.pending_publications(limit=32, after=after)
|
||||
if not page:
|
||||
return
|
||||
yield from page
|
||||
last = page[-1]
|
||||
after = (last.created_at_utc, last.job_id)
|
||||
if len(page) < 32:
|
||||
return
|
||||
|
||||
|
||||
def _timestamp(value: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise ObservatoryRecordedQueueError(
|
||||
"recorded publication timestamp is invalid"
|
||||
) from exc
|
||||
raise ObservatoryRecordedQueueError("recorded publication timestamp is invalid") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise ObservatoryRecordedQueueError(
|
||||
"recorded publication timestamp has no timezone"
|
||||
)
|
||||
raise ObservatoryRecordedQueueError("recorded publication timestamp has no timezone")
|
||||
return parsed
|
||||
|
||||
@@ -160,6 +160,19 @@ class PortableRecordedQueueBindingService:
|
||||
source_service = self._source_service(portable)
|
||||
return self._bind(recorded, source_service.check(source_session_id))
|
||||
|
||||
def prepare_check(
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> PortableRecordedRunPreparation:
|
||||
"""Prepare missing camera metadata, then seal the usual non-persistent check."""
|
||||
|
||||
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
|
||||
source_service = self._source_service(portable)
|
||||
return self._bind(recorded, source_service.prepare_check(source_session_id))
|
||||
|
||||
def admit(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -25,7 +25,18 @@ from k1link.laboratory.canonical_rerun_overlay import (
|
||||
canonical_lab_replay,
|
||||
canonical_recording_id,
|
||||
)
|
||||
from k1link.observatory.portable_object_replay import (
|
||||
RENDERER_VERSION as OBJECT_RENDERER_VERSION,
|
||||
)
|
||||
from k1link.observatory.portable_object_replay import load_object_data, log_objects
|
||||
from k1link.observatory.portable_result_view import PortableResultViewService
|
||||
from k1link.observatory.portable_semantic_replay import (
|
||||
RENDERER_VERSION as SEMANTIC_RENDERER_VERSION,
|
||||
)
|
||||
from k1link.observatory.portable_semantic_replay import (
|
||||
load_semantic_data,
|
||||
log_semantics,
|
||||
)
|
||||
from k1link.observatory.portable_tgs_replay import (
|
||||
PortableReplayError,
|
||||
load_tgs_data,
|
||||
@@ -39,7 +50,11 @@ from k1link.sessions.media import RecordedMediaEpoch, RecordedMediaInspector
|
||||
|
||||
RENDERER_VERSION = "portable-tgs-camera-rerun-0.36.3-v1"
|
||||
_SHA = re.compile(r"^[a-f0-9]{64}$")
|
||||
_RESULT = re.compile(r"^m49-tgs-portable-review-[a-f0-9]{64}$")
|
||||
_RESULT = re.compile(
|
||||
r"^(?:m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
|
||||
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance))-[a-f0-9]{64}$"
|
||||
)
|
||||
_COMPOSITION = re.compile(r"^ai-composition-[a-f0-9]{64}$")
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@@ -65,7 +80,16 @@ class PortableReplayService:
|
||||
if _RESULT.fullmatch(result_id) is None or _SHA.fullmatch(base_sha) is None:
|
||||
raise PortableReplayError("unsupported replay identity")
|
||||
view = cast(dict[str, Any], self.view.read(result_id))
|
||||
identity = [RENDERER_VERSION, result_id, base_sha, view["artifact_manifest_id"]]
|
||||
module = view.get("result_document", {}).get("module", {})
|
||||
module_id = module.get("module_id") if isinstance(module, dict) else None
|
||||
renderer = (
|
||||
OBJECT_RENDERER_VERSION
|
||||
if module_id in {"rf-detr", "object-distance"}
|
||||
else SEMANTIC_RENDERER_VERSION
|
||||
if result_id.startswith(("lab-v1-", "ai-layer-"))
|
||||
else RENDERER_VERSION
|
||||
)
|
||||
identity = [renderer, result_id, base_sha, view["artifact_manifest_id"]]
|
||||
return hashlib.sha256("\0".join(identity).encode()).hexdigest(), view
|
||||
|
||||
def cached(self, result_id: str, base_sha: str) -> CanonicalLabReplayArtifact | None:
|
||||
@@ -141,15 +165,35 @@ class PortableReplayService:
|
||||
):
|
||||
raise PortableReplayError("source catalog changed since calculation")
|
||||
epoch = self._camera(view["source_session_id"], bundle)
|
||||
data = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
|
||||
starts = [epoch.timeline_start_seconds] + [
|
||||
epoch.timeline_start_seconds + part.end_time_seconds for part in epoch.segments[:-1]
|
||||
]
|
||||
if len(starts) != len(data.rows) or any(
|
||||
abs(start - row["session_seconds"]) > 1e-8
|
||||
for start, row in zip(starts, data.rows, strict=True)
|
||||
):
|
||||
raise PortableReplayError("camera and costmap anchor clocks disagree")
|
||||
tgs = None
|
||||
semantics = None
|
||||
objects = None
|
||||
if result_id.startswith("m49-tgs-portable-review-"):
|
||||
tgs = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
|
||||
if len(starts) != len(tgs.rows) or any(
|
||||
abs(start - row["session_seconds"]) > 1e-8
|
||||
for start, row in zip(starts, tgs.rows, strict=True)
|
||||
):
|
||||
raise PortableReplayError("camera and costmap anchor clocks disagree")
|
||||
elif result_id.startswith(("ai-layer-rf-detr-", "ai-layer-object-distance-")):
|
||||
objects = load_object_data(
|
||||
view,
|
||||
self.view.artifacts,
|
||||
source_bundle_sha256=bundle_sha,
|
||||
starts=starts,
|
||||
end_seconds=epoch.timeline_end_seconds,
|
||||
)
|
||||
else:
|
||||
semantics = load_semantic_data(
|
||||
view,
|
||||
self.view.artifacts,
|
||||
source_bundle_sha256=bundle_sha,
|
||||
starts=starts,
|
||||
end_seconds=epoch.timeline_end_seconds,
|
||||
)
|
||||
recording_id = canonical_recording_id(base[0])
|
||||
self.cache.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if self.cache.is_symlink() or shutil.disk_usage(self.cache).free < 3 * 1024**3:
|
||||
@@ -162,7 +206,12 @@ class PortableReplayService:
|
||||
try:
|
||||
recording.set_sinks(rr.FileSink(output, write_footer=True))
|
||||
_log_video(recording, proxy, epoch)
|
||||
log_tgs(recording, data, epoch.timeline_end_seconds)
|
||||
if tgs is not None:
|
||||
log_tgs(recording, tgs, epoch.timeline_end_seconds)
|
||||
if semantics is not None:
|
||||
log_semantics(recording, semantics)
|
||||
if objects is not None:
|
||||
log_objects(recording, objects)
|
||||
recording.flush(timeout_sec=180)
|
||||
finally:
|
||||
recording.disconnect()
|
||||
@@ -188,6 +237,191 @@ class PortableReplayService:
|
||||
os.replace(staged, self.cache / f"{key}.json")
|
||||
return artifact
|
||||
|
||||
def _composition_key(
|
||||
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
|
||||
) -> tuple[str, tuple[dict[str, Any], ...]]:
|
||||
if _COMPOSITION.fullmatch(run_id) is None or _SHA.fullmatch(base_sha) is None:
|
||||
raise PortableReplayError("unsupported composition replay identity")
|
||||
if not result_ids or len(result_ids) != len(set(result_ids)):
|
||||
raise PortableReplayError("composition replay members are invalid")
|
||||
views = tuple(cast(dict[str, Any], self.view.read(result_id)) for result_id in result_ids)
|
||||
source_ids = {view["source_session_id"] for view in views}
|
||||
if len(source_ids) != 1:
|
||||
raise PortableReplayError("composition replay members use different sources")
|
||||
identity = [
|
||||
"portable-composition-rerun-0.36.3-v1",
|
||||
run_id,
|
||||
base_sha,
|
||||
*[f"{view['result_id']}:{view['artifact_manifest_id']}" for view in views],
|
||||
]
|
||||
return hashlib.sha256("\0".join(identity).encode()).hexdigest(), views
|
||||
|
||||
def cached_composition(
|
||||
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
|
||||
) -> CanonicalLabReplayArtifact | None:
|
||||
key, _ = self._composition_key(run_id, result_ids, base_sha)
|
||||
sidecar = self.cache / f"{key}.json"
|
||||
if not sidecar.exists():
|
||||
return None
|
||||
metadata = read_json(sidecar, 8192)
|
||||
digest = metadata.get("sha256")
|
||||
name = metadata.get("file")
|
||||
if (
|
||||
metadata.get("key") != key
|
||||
or not isinstance(digest, str)
|
||||
or _SHA.fullmatch(digest) is None
|
||||
or not isinstance(name, str)
|
||||
or re.fullmatch(r"[a-f0-9]{64}\.replay\.rrd", name) is None
|
||||
):
|
||||
raise PortableReplayError("composition replay cache identity changed")
|
||||
path = self.cache / name
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not path.is_file()
|
||||
or path.stat().st_size != metadata["byte_length"]
|
||||
):
|
||||
raise PortableReplayError("composition replay cache length changed")
|
||||
if not 4 <= metadata["byte_length"] <= 1024 * 1024 * 1024:
|
||||
raise PortableReplayError("composition replay cache exceeds bounds")
|
||||
stamp = stat_identity(path.stat())
|
||||
fingerprint = (path, digest)
|
||||
if self._verified.get(fingerprint) != stamp:
|
||||
with path.open("rb") as stream:
|
||||
if hashlib.file_digest(stream, "sha256").hexdigest() != digest:
|
||||
raise PortableReplayError("composition replay cache digest changed")
|
||||
self._verified[fingerprint] = stamp
|
||||
while len(self._verified) > 32:
|
||||
self._verified.popitem(last=False)
|
||||
return CanonicalLabReplayArtifact(path, metadata["byte_length"], digest)
|
||||
|
||||
def prepare_composition(
|
||||
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
|
||||
) -> CanonicalLabReplayArtifact:
|
||||
with _LOCK:
|
||||
cached = self.cached_composition(run_id, result_ids, base_sha)
|
||||
if cached is not None:
|
||||
return cached
|
||||
key, views = self._composition_key(run_id, result_ids, base_sha)
|
||||
source_session_id = cast(str, views[0]["source_session_id"])
|
||||
base = self.recording_source(source_session_id)
|
||||
if base is None or base[1] != base_sha:
|
||||
raise PortableReplayError("composition source recording is not ready")
|
||||
recording_id = canonical_recording_id(base[0])
|
||||
epoch: RecordedMediaEpoch | None = None
|
||||
tgs_rows: list[Any] = []
|
||||
semantic_rows: list[Any] = []
|
||||
object_rows: list[Any] = []
|
||||
for view in views:
|
||||
result_id = cast(str, view["result_id"])
|
||||
binding = self.view.sessions.get_lab_instance(result_id)
|
||||
if binding is None or binding.source_session_id != source_session_id:
|
||||
raise PortableReplayError("composition result binding disappeared")
|
||||
source = binding.provenance["source"]
|
||||
bundle_sha = source["bundle_sha256"]
|
||||
if not isinstance(bundle_sha, str) or _SHA.fullmatch(bundle_sha) is None:
|
||||
raise PortableReplayError("composition source bundle identity is invalid")
|
||||
bundle_path = (
|
||||
self.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY / f"{bundle_sha}.json"
|
||||
)
|
||||
bundle = read_json(bundle_path, 32 * 1024 * 1024)
|
||||
verified_file(bundle_path, bundle_sha, bundle_path.stat().st_size)
|
||||
if bundle["source_session_id"] != source_session_id:
|
||||
raise PortableReplayError("composition member belongs to another session")
|
||||
_, catalog_sha = self.view.sessions.get_session_with_catalog_snapshot(
|
||||
source_session_id
|
||||
)
|
||||
if (
|
||||
catalog_sha != source["catalog_sha256"]
|
||||
or catalog_sha != bundle["source_catalog_sha256"]
|
||||
):
|
||||
raise PortableReplayError("composition source catalog changed")
|
||||
member_epoch = self._camera(source_session_id, bundle)
|
||||
if epoch is None:
|
||||
epoch = member_epoch
|
||||
elif (
|
||||
epoch.timeline_start_seconds != member_epoch.timeline_start_seconds
|
||||
or epoch.timeline_end_seconds != member_epoch.timeline_end_seconds
|
||||
or len(epoch.segments) != len(member_epoch.segments)
|
||||
):
|
||||
raise PortableReplayError("composition member clocks disagree")
|
||||
starts = [member_epoch.timeline_start_seconds] + [
|
||||
member_epoch.timeline_start_seconds + part.end_time_seconds
|
||||
for part in member_epoch.segments[:-1]
|
||||
]
|
||||
if result_id.startswith("m49-tgs-portable-review-"):
|
||||
data = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
|
||||
if len(starts) != len(data.rows) or any(
|
||||
abs(start - row["session_seconds"]) > 1e-8
|
||||
for start, row in zip(starts, data.rows, strict=True)
|
||||
):
|
||||
raise PortableReplayError("composition camera and TGS clocks disagree")
|
||||
tgs_rows.append(data)
|
||||
elif result_id.startswith(("ai-layer-rf-detr-", "ai-layer-object-distance-")):
|
||||
object_rows.append(
|
||||
load_object_data(
|
||||
view,
|
||||
self.view.artifacts,
|
||||
source_bundle_sha256=bundle_sha,
|
||||
starts=starts,
|
||||
end_seconds=member_epoch.timeline_end_seconds,
|
||||
)
|
||||
)
|
||||
else:
|
||||
semantic_rows.append(
|
||||
load_semantic_data(
|
||||
view,
|
||||
self.view.artifacts,
|
||||
source_bundle_sha256=bundle_sha,
|
||||
starts=starts,
|
||||
end_seconds=member_epoch.timeline_end_seconds,
|
||||
)
|
||||
)
|
||||
if epoch is None:
|
||||
raise PortableReplayError("composition has no replay clock")
|
||||
self.cache.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if self.cache.is_symlink() or shutil.disk_usage(self.cache).free < 3 * 1024**3:
|
||||
raise PortableReplayError("replay cache has insufficient safe space")
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix=".pack-composition-", dir=self.cache
|
||||
) as temporary:
|
||||
root = Path(temporary)
|
||||
proxy = self._video(epoch, root)
|
||||
output = root / "overlay.rrd"
|
||||
recording = rr.RecordingStream(APPLICATION_ID, recording_id=recording_id)
|
||||
try:
|
||||
recording.set_sinks(rr.FileSink(output, write_footer=True))
|
||||
_log_video(recording, proxy, epoch)
|
||||
for data in tgs_rows:
|
||||
log_tgs(recording, data, epoch.timeline_end_seconds)
|
||||
for data in semantic_rows:
|
||||
log_semantics(recording, data)
|
||||
for data in object_rows:
|
||||
log_objects(recording, data)
|
||||
recording.flush(timeout_sec=180)
|
||||
finally:
|
||||
recording.disconnect()
|
||||
with output.open("rb") as stream:
|
||||
digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
artifact = canonical_lab_replay(
|
||||
base[0],
|
||||
base_generation_sha256=base_sha,
|
||||
overlay=CanonicalLabOverlayArtifact(output, output.stat().st_size, digest),
|
||||
result_id=run_id,
|
||||
recording_id=recording_id,
|
||||
cache_root=self.cache,
|
||||
)
|
||||
metadata = {
|
||||
"key": key,
|
||||
"file": artifact.path.name,
|
||||
"sha256": artifact.sha256,
|
||||
"byte_length": artifact.byte_length,
|
||||
}
|
||||
staged = self.cache / f".{key}.json"
|
||||
staged.write_text(json.dumps(metadata, sort_keys=True), encoding="utf-8")
|
||||
os.chmod(staged, 0o600)
|
||||
os.replace(staged, self.cache / f"{key}.json")
|
||||
return artifact
|
||||
|
||||
def _camera(self, session_id: str, bundle: dict[str, Any]) -> RecordedMediaEpoch:
|
||||
camera = bundle["camera"]
|
||||
artifact = self.view.sessions.get_recorded_media(session_id, camera["artifact_id"])
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Project sealed LAB V1 masks, one native frame at a time, without inference.
|
||||
|
||||
No archive LAB, model installation, or Worker is consulted. The source camera
|
||||
clock and the published component identities are the only evidence inputs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import tarfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import numpy as np
|
||||
import rerun as rr
|
||||
from PIL import Image
|
||||
|
||||
from k1link.artifact_gateway import CentralArtifactStore
|
||||
from k1link.laboratory.canonical_rerun_overlay import (
|
||||
_encoded_semantic_png,
|
||||
_localized_semantic_label,
|
||||
_semantic_palette,
|
||||
)
|
||||
from k1link.observatory.portable_tgs_replay import PortableReplayError, read_json, verified_file
|
||||
|
||||
RESULT_SCHEMA = "missioncore.recorded-eomt-ddrnet-review/v2"
|
||||
RENDERER_VERSION = "portable-eomt-ddrnet-camera-rerun-0.36.3-v1"
|
||||
# The target labels belong to this exact admitted EoMT preprocessing profile,
|
||||
# not to the model's raw Cityscapes label order. A new profile needs an adapter.
|
||||
EOMT_PROFILE_SHA = "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||
EOMT_LABELS = (
|
||||
"outside_valid_fov",
|
||||
"person",
|
||||
"bicycle",
|
||||
"motorcycle",
|
||||
"car",
|
||||
"heavy_vehicle",
|
||||
"building_structure",
|
||||
"paved_road",
|
||||
"sidewalk_curb",
|
||||
"ground_dirt",
|
||||
"grass_low_vegetation",
|
||||
"tree_woody_vegetation",
|
||||
"sky",
|
||||
"static_obstacle",
|
||||
"animal",
|
||||
"other_background",
|
||||
)
|
||||
_PNG_BOUND = 1024 * 1024
|
||||
_MAX_FRAMES = 250_000
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SemanticReplayData:
|
||||
times: tuple[float, ...]
|
||||
end_seconds: float
|
||||
city_masks: Path | None
|
||||
vegetation_masks: Path | None
|
||||
classes: dict[str, list[dict[str, Any]]]
|
||||
|
||||
|
||||
def load_semantic_data(
|
||||
view: dict[str, Any],
|
||||
store: CentralArtifactStore,
|
||||
*,
|
||||
source_bundle_sha256: str,
|
||||
starts: list[float],
|
||||
end_seconds: float,
|
||||
) -> SemanticReplayData:
|
||||
doc = view["result_document"]
|
||||
source = doc["source"]
|
||||
if (
|
||||
doc.get("schema_version")
|
||||
not in {
|
||||
RESULT_SCHEMA,
|
||||
"missioncore.recorded-ai-layer-review/v1",
|
||||
}
|
||||
or doc.get("result_id") != view["result_id"]
|
||||
or source.get("session_id") != view["source_session_id"]
|
||||
or source.get("bundle_sha256") != source_bundle_sha256
|
||||
or not 0 < len(starts) <= _MAX_FRAMES
|
||||
or source.get("frame_count") != len(starts)
|
||||
or source.get("timeline_start_seconds") != starts[0]
|
||||
or source.get("timeline_end_seconds") != end_seconds
|
||||
or not np.isfinite([*starts, end_seconds]).all()
|
||||
or np.any(np.diff([*starts, end_seconds]) <= 0)
|
||||
):
|
||||
raise PortableReplayError("semantic result and source clock disagree")
|
||||
members = {row["role"]: row for row in view["artifacts"]}
|
||||
if len(members) != len(view["artifacts"]):
|
||||
raise PortableReplayError("semantic artifact roles are duplicated")
|
||||
|
||||
def artifact(role: str, media: str) -> Path:
|
||||
member = members.get(role)
|
||||
if member is None:
|
||||
raise PortableReplayError("semantic artifact is missing")
|
||||
if member["media_type"] != media:
|
||||
raise PortableReplayError("semantic artifact media type changed")
|
||||
return verified_file(
|
||||
store.object_path(member["sha256"]),
|
||||
member["sha256"],
|
||||
member["byte_length"],
|
||||
)
|
||||
|
||||
schema = doc["schema_version"]
|
||||
module = doc.get("module")
|
||||
module_id = module.get("module_id") if isinstance(module, dict) else None
|
||||
include_eomt = schema == RESULT_SCHEMA or module_id == "eomt"
|
||||
include_ddrnet = schema == RESULT_SCHEMA or module_id == "ddrnet"
|
||||
if not include_eomt and not include_ddrnet:
|
||||
raise PortableReplayError("semantic AI module is unsupported")
|
||||
|
||||
city_classes: list[dict[str, Any]] = []
|
||||
city_masks: Path | None = None
|
||||
if include_eomt:
|
||||
city = read_json(artifact("eomt-result-document", "application/json"))
|
||||
if schema == RESULT_SCHEMA:
|
||||
bound = doc["components"]["eomt"]
|
||||
component_bound = (
|
||||
bound["result_id"] == city["result_id"]
|
||||
and bound["frames_processed"] == len(starts)
|
||||
and bound["result_document_sha256"] == members["eomt-result-document"]["sha256"]
|
||||
)
|
||||
else:
|
||||
component_bound = (
|
||||
doc["module"].get("component_result_sha256")
|
||||
== members["eomt-result-document"]["sha256"]
|
||||
)
|
||||
if not component_bound or (
|
||||
city["identity"]["configuration"].get("profile_sha256") != EOMT_PROFILE_SHA
|
||||
or city.get("frames_processed") != len(starts)
|
||||
or city.get("session_id") != view["source_session_id"]
|
||||
or city.get("input_sha256") != source["camera_input_sha256"]
|
||||
or city.get("timestamp_basis") != "session-time-seconds"
|
||||
):
|
||||
raise PortableReplayError("EoMT target profile or source changed")
|
||||
for class_id, label in enumerate(EOMT_LABELS):
|
||||
color = hashlib.sha256(f"mission-core-segment-{class_id}".encode()).digest()
|
||||
city_classes.append(
|
||||
{
|
||||
"class_id": class_id,
|
||||
"label": label,
|
||||
"color_rgb": [64 + color[channel] % 176 for channel in range(3)],
|
||||
}
|
||||
)
|
||||
frames = artifact("eomt-panoptic-frame-metadata", "application/x-ndjson")
|
||||
with frames.open("rb") as stream:
|
||||
for index, timestamp in enumerate(starts):
|
||||
line = stream.readline(_PNG_BOUND + 1)
|
||||
if not line or len(line) > _PNG_BOUND:
|
||||
raise PortableReplayError("semantic frame metadata is truncated or oversized")
|
||||
row = json.loads(line)
|
||||
value = row.get("session_seconds")
|
||||
if (
|
||||
row.get("frame_index") != index
|
||||
or row.get("sequence") != index + 1
|
||||
or type(value) not in (int, float)
|
||||
or not np.isfinite(value)
|
||||
or abs(value - timestamp) > 1e-8
|
||||
):
|
||||
raise PortableReplayError("semantic frame clock disagrees with camera")
|
||||
for category in row["semantic_classes"]:
|
||||
class_id = category.get("id")
|
||||
if (
|
||||
type(class_id) is not int
|
||||
or not 0 < class_id < len(EOMT_LABELS)
|
||||
or category.get("label") != EOMT_LABELS[class_id]
|
||||
):
|
||||
raise PortableReplayError("EoMT metadata taxonomy changed")
|
||||
if stream.read(1):
|
||||
raise PortableReplayError("semantic metadata contains extra frames")
|
||||
city_masks = artifact("eomt-panoptic-mask-archive", "application/gzip")
|
||||
|
||||
vegetation_classes: list[dict[str, Any]] = []
|
||||
vegetation_masks: Path | None = None
|
||||
if include_ddrnet:
|
||||
vegetation = read_json(artifact("ddrnet-result-document", "application/json"))
|
||||
if schema == RESULT_SCHEMA:
|
||||
bound = doc["components"]["ddrnet"]
|
||||
component_bound = (
|
||||
bound["result_id"] == vegetation["result_id"]
|
||||
and bound["frames_processed"] == len(starts)
|
||||
and bound["result_document_sha256"] == members["ddrnet-result-document"]["sha256"]
|
||||
)
|
||||
else:
|
||||
component_bound = (
|
||||
doc["module"].get("component_result_sha256")
|
||||
== members["ddrnet-result-document"]["sha256"]
|
||||
)
|
||||
semantic = vegetation["video_semantics"]
|
||||
taxonomy = semantic["taxonomy"]
|
||||
vegetation_classes = taxonomy["classes"]
|
||||
archive = semantic["mask_archive"]
|
||||
packaged = members.get("ddrnet-semantic-mask-archive")
|
||||
if (
|
||||
not component_bound
|
||||
or packaged is None
|
||||
or taxonomy.get("schema_version") != "missioncore.lab-v1-vegetation-taxonomy/v1"
|
||||
or not isinstance(vegetation_classes, list)
|
||||
or {row["class_id"] for row in vegetation_classes} != set(range(64))
|
||||
or archive.get("frame_count") != len(starts)
|
||||
or (archive.get("width"), archive.get("height")) != (800, 600)
|
||||
or archive.get("encoding") != "uint8-class-id-png"
|
||||
or any(
|
||||
archive.get(key) != packaged[key] for key in ("sha256", "byte_length", "media_type")
|
||||
)
|
||||
):
|
||||
raise PortableReplayError("DDRNet mask taxonomy or archive binding changed")
|
||||
_semantic_palette(cast(list[object], vegetation_classes))
|
||||
for row in vegetation_classes:
|
||||
if not isinstance(row.get("label"), str) or not row["label"]:
|
||||
raise PortableReplayError("DDRNet class label is invalid")
|
||||
vegetation_masks = artifact("ddrnet-semantic-mask-archive", "application/zip")
|
||||
return SemanticReplayData(
|
||||
tuple(starts),
|
||||
end_seconds,
|
||||
city_masks,
|
||||
vegetation_masks,
|
||||
{
|
||||
**({"city": city_classes} if include_eomt else {}),
|
||||
**({"vegetation": vegetation_classes} if include_ddrnet else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def log_semantics(recording: rr.RecordingStream, data: SemanticReplayData) -> None:
|
||||
try:
|
||||
_log_semantics(recording, data)
|
||||
except (tarfile.TarError, zipfile.BadZipFile, EOFError) as exc:
|
||||
raise PortableReplayError("semantic mask archive is damaged") from exc
|
||||
|
||||
|
||||
def _log_semantics(recording: rr.RecordingStream, data: SemanticReplayData) -> None:
|
||||
"""No whole-route decoded tensor or extracted archive; release each frame."""
|
||||
palettes = {
|
||||
layer: _semantic_palette(cast(list[object], classes))
|
||||
for layer, classes in data.classes.items()
|
||||
}
|
||||
for layer, classes in data.classes.items():
|
||||
recording.log(
|
||||
f"/perception/camera/segmentation/{layer}",
|
||||
rr.AnnotationContext(
|
||||
[
|
||||
rr.ClassDescription(
|
||||
info=rr.AnnotationInfo(
|
||||
id=row["class_id"],
|
||||
label=_localized_semantic_label(row["label"]),
|
||||
color=[*row["color_rgb"], 255],
|
||||
)
|
||||
)
|
||||
for row in classes
|
||||
]
|
||||
),
|
||||
static=True,
|
||||
)
|
||||
|
||||
def frame(layer: str, index: int, payload: bytes) -> None:
|
||||
with Image.open(io.BytesIO(payload)) as image:
|
||||
if image.size != (800, 600) or image.mode not in ("L", "P") or image.format != "PNG":
|
||||
raise PortableReplayError("semantic mask raster changed")
|
||||
mask = np.asarray(image, dtype=np.uint8)
|
||||
if int(mask.max()) >= len(data.classes[layer]):
|
||||
raise PortableReplayError("semantic mask contains an unknown class")
|
||||
encoded = _encoded_semantic_png(mask, palettes[layer])
|
||||
recording.set_time(
|
||||
"session_time",
|
||||
duration=np.timedelta64(round(data.times[index] * 1e9), "ns"),
|
||||
)
|
||||
recording.log(
|
||||
f"/perception/camera/segmentation/{layer}",
|
||||
rr.EncodedImage(contents=encoded, media_type="image/png", opacity=0.72, draw_order=1.0),
|
||||
)
|
||||
|
||||
if data.city_masks is not None:
|
||||
seen: set[int] = set()
|
||||
root_seen = False
|
||||
# Stream gzip once. Members may arrive in archive order rather than frame
|
||||
# order; every log uses its own verified camera timestamp.
|
||||
with tarfile.open(data.city_masks, "r|gz") as archive:
|
||||
for member in archive:
|
||||
if member.isdir() and member.name.rstrip("/") == "semantic-masks":
|
||||
if root_seen or member.size != 0:
|
||||
raise PortableReplayError("EoMT mask archive directory is invalid")
|
||||
root_seen = True
|
||||
cast(Any, archive).members.clear()
|
||||
continue
|
||||
match = re.fullmatch(r"semantic-masks/frame-([0-9]{6})\.png", member.name)
|
||||
if not match or not member.isfile() or not 0 < member.size <= _PNG_BOUND:
|
||||
raise PortableReplayError("EoMT mask archive member is unsafe")
|
||||
index = int(match[1]) - 1
|
||||
if not 0 <= index < len(data.times) or index in seen:
|
||||
raise PortableReplayError(
|
||||
"EoMT mask sequence is duplicated or outside coverage"
|
||||
)
|
||||
source = archive.extractfile(member)
|
||||
if source is None:
|
||||
raise PortableReplayError("EoMT mask is unavailable")
|
||||
with source:
|
||||
payload = source.read(_PNG_BOUND + 1)
|
||||
if len(payload) != member.size:
|
||||
raise PortableReplayError("EoMT mask is truncated")
|
||||
frame("city", index, payload)
|
||||
seen.add(index)
|
||||
# tarfile otherwise retains a TarInfo for every frame even in
|
||||
# streaming mode on Python 3.12. No lookup uses that history.
|
||||
cast(Any, archive).members.clear()
|
||||
if len(seen) != len(data.times):
|
||||
raise PortableReplayError("EoMT mask sequence is incomplete")
|
||||
if data.vegetation_masks is not None:
|
||||
with zipfile.ZipFile(data.vegetation_masks) as masks:
|
||||
names = masks.namelist()
|
||||
expected = {f"masks/frame-{index + 1:06d}.png" for index in range(len(data.times))}
|
||||
if len(names) != len(expected) or set(names) != expected:
|
||||
raise PortableReplayError("DDRNet mask sequence is incomplete or duplicated")
|
||||
for index in range(len(data.times)):
|
||||
info = masks.getinfo(f"masks/frame-{index + 1:06d}.png")
|
||||
if not 0 < info.file_size <= _PNG_BOUND:
|
||||
raise PortableReplayError("DDRNet mask exceeds the frame bound")
|
||||
frame("vegetation", index, masks.read(info))
|
||||
recording.set_time(
|
||||
"session_time",
|
||||
duration=np.timedelta64(round(data.end_seconds * 1e9), "ns"),
|
||||
)
|
||||
recording.log("/perception/camera/segmentation", rr.Clear(recursive=True))
|
||||
@@ -36,7 +36,15 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
|
||||
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
|
||||
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
|
||||
PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2"
|
||||
PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
|
||||
PORTABLE_M49_DISPLAY_NAME: Final = "TRAVEL TGS"
|
||||
AI_DDRNET_SETUP_ID: Final = "ai-segmentation-ddrnet-v1"
|
||||
AI_EOMT_SETUP_ID: Final = "ai-segmentation-eomt-v1"
|
||||
AI_RF_DETR_SETUP_ID: Final = "ai-detection-rf-detr-v1"
|
||||
AI_OBJECT_DISTANCE_SETUP_ID: Final = "ai-range-object-distance-v1"
|
||||
AI_DDRNET_DISPLAY_NAME: Final = "DDRNet-39 · GOOSE"
|
||||
AI_EOMT_DISPLAY_NAME: Final = "EoMT Large · Cityscapes"
|
||||
AI_RF_DETR_DISPLAY_NAME: Final = "RF-DETR Large"
|
||||
AI_OBJECT_DISTANCE_DISPLAY_NAME: Final = "Дистанция до объектов · K1 LiDAR"
|
||||
|
||||
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
_OBSERVATION_ONLY_AUTHORITY: Final = {
|
||||
@@ -63,6 +71,38 @@ _SETUP_PRESENTATION: Final = {
|
||||
"incompatible": "Запись не соответствует требованиям TRAVEL TGS.",
|
||||
"executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.",
|
||||
},
|
||||
AI_DDRNET_SETUP_ID: {
|
||||
"lab_id": "LAB AI-DDRNET",
|
||||
"display_name": AI_DDRNET_DISPLAY_NAME,
|
||||
"description": "Независимая семантическая сегментация DDRNet записанной K1-сессии.",
|
||||
"compatible": "Запись соответствует требованиям DDRNet.",
|
||||
"incompatible": "Запись не соответствует требованиям DDRNet.",
|
||||
"executor_unavailable": "AI-модуль DDRNet на Worker 006 недоступен.",
|
||||
},
|
||||
AI_EOMT_SETUP_ID: {
|
||||
"lab_id": "LAB AI-EOMT",
|
||||
"display_name": AI_EOMT_DISPLAY_NAME,
|
||||
"description": "Независимая семантическая сегментация EoMT записанной K1-сессии.",
|
||||
"compatible": "Запись соответствует требованиям EoMT.",
|
||||
"incompatible": "Запись не соответствует требованиям EoMT.",
|
||||
"executor_unavailable": "AI-модуль EoMT на Worker 006 недоступен.",
|
||||
},
|
||||
AI_RF_DETR_SETUP_ID: {
|
||||
"lab_id": "LAB AI-RF-DETR",
|
||||
"display_name": AI_RF_DETR_DISPLAY_NAME,
|
||||
"description": "Независимая детекция объектов RF-DETR на записанном видео.",
|
||||
"compatible": "Запись соответствует требованиям RF-DETR.",
|
||||
"incompatible": "Запись не соответствует требованиям RF-DETR.",
|
||||
"executor_unavailable": "AI-модуль RF-DETR на Worker 006 недоступен.",
|
||||
},
|
||||
AI_OBJECT_DISTANCE_SETUP_ID: {
|
||||
"lab_id": "LAB AI-RANGE",
|
||||
"display_name": AI_OBJECT_DISTANCE_DISPLAY_NAME,
|
||||
"description": ("RF-DETR и синхронное облако точек для оценки дистанции до объектов."),
|
||||
"compatible": "Запись содержит видео, облако точек, позу и калибровку.",
|
||||
"incompatible": "Для дистанции не хватает видео или пространственных данных.",
|
||||
"executor_unavailable": "AI-модуль дистанции на Worker 006 недоступен.",
|
||||
},
|
||||
}
|
||||
|
||||
_MODEL_PRESENTATION: Final = {
|
||||
|
||||
@@ -21,6 +21,10 @@ from k1link.observatory.m49_portable_result import (
|
||||
M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validate_m49_portable_result,
|
||||
)
|
||||
from k1link.observatory.modular_result import (
|
||||
MODULAR_RESULT_CONTRACT_SHA256,
|
||||
validate_modular_result,
|
||||
)
|
||||
from k1link.observatory.portable_artifact_transport import (
|
||||
PortableArtifactTransportError,
|
||||
PortableObservatoryArtifactTransport,
|
||||
@@ -55,9 +59,7 @@ OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_
|
||||
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
|
||||
)
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
|
||||
)
|
||||
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
|
||||
|
||||
|
||||
class PortableWorkerIntegrationError(RuntimeError):
|
||||
@@ -162,6 +164,10 @@ _BUILTIN_VALIDATOR_REGISTRATIONS: Final = (
|
||||
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_m49_portable_result,
|
||||
),
|
||||
PortableResultContractValidatorRegistration(
|
||||
contract_sha256=MODULAR_RESULT_CONTRACT_SHA256,
|
||||
validator=validate_modular_result,
|
||||
),
|
||||
)
|
||||
_BUILTIN_VALIDATORS_BY_CONTRACT: Final = {
|
||||
registration.contract_sha256: registration.validator
|
||||
@@ -172,26 +178,23 @@ _BUILTIN_VALIDATORS_BY_CONTRACT: Final = {
|
||||
def portable_result_validator_registry(
|
||||
definitions: PortableRunDefinitionRegistry,
|
||||
*,
|
||||
registrations: tuple[PortableResultContractValidatorRegistration, ...]
|
||||
| None = None,
|
||||
registrations: tuple[PortableResultContractValidatorRegistration, ...] | None = None,
|
||||
) -> PortableResultContractValidatorRegistry:
|
||||
"""Select server-installed validators by exact result-contract identity."""
|
||||
|
||||
installed = (
|
||||
_BUILTIN_VALIDATOR_REGISTRATIONS
|
||||
if registrations is None
|
||||
else registrations
|
||||
)
|
||||
installed = _BUILTIN_VALIDATOR_REGISTRATIONS if registrations is None else registrations
|
||||
by_contract = {
|
||||
registration.contract_sha256: registration
|
||||
for registration in PortableResultContractValidatorRegistry(installed).registrations
|
||||
}
|
||||
selected = tuple(
|
||||
by_contract[definition.result_contract.contract_sha256]
|
||||
for definition in definitions.definitions
|
||||
if definition.result_contract.contract_sha256 in by_contract
|
||||
)
|
||||
registry = PortableResultContractValidatorRegistry(selected)
|
||||
selected: list[PortableResultContractValidatorRegistration] = []
|
||||
selected_contracts: set[str] = set()
|
||||
for definition in definitions.definitions:
|
||||
contract_sha256 = definition.result_contract.contract_sha256
|
||||
if contract_sha256 in by_contract and contract_sha256 not in selected_contracts:
|
||||
selected.append(by_contract[contract_sha256])
|
||||
selected_contracts.add(contract_sha256)
|
||||
registry = PortableResultContractValidatorRegistry(tuple(selected))
|
||||
for definition in definitions.definitions:
|
||||
if definition.executor.ready:
|
||||
try:
|
||||
@@ -287,8 +290,7 @@ def build_portable_observatory_worker_integration(
|
||||
definition.setup_id
|
||||
for definition in definitions.definitions
|
||||
if any(
|
||||
registration.contract_sha256
|
||||
== definition.result_contract.contract_sha256
|
||||
registration.contract_sha256 == definition.result_contract.contract_sha256
|
||||
for registration in validator_registry.registrations
|
||||
)
|
||||
),
|
||||
|
||||
@@ -575,9 +575,7 @@ class ObservatoryRecordedJob:
|
||||
self.claim_heartbeat_at_utc,
|
||||
)
|
||||
if (self.active_claim_token is None) != (self.active_claimant_id is None):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job claim ownership is partial"
|
||||
)
|
||||
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim ownership is partial")
|
||||
if self.active_claim_token is None:
|
||||
if any(value is not None for value in lease_values) or self.claim_renewal_count != 0:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
@@ -654,9 +652,7 @@ class ObservatoryRecordedJob:
|
||||
and self.publication_error is None
|
||||
and self.published_at_utc is None
|
||||
),
|
||||
"pending": (
|
||||
self.publication_error is None and self.published_at_utc is None
|
||||
),
|
||||
"pending": (self.publication_error is None and self.published_at_utc is None),
|
||||
"failed": (
|
||||
self.publication_attempts >= 1
|
||||
and self.publication_error is not None
|
||||
@@ -673,9 +669,7 @@ class ObservatoryRecordedJob:
|
||||
"recorded-job publication receipt is inconsistent"
|
||||
)
|
||||
if self.publication_state != "not-required" and (
|
||||
self.state != "succeeded"
|
||||
or self.result_id is None
|
||||
or self.result_sha256 is None
|
||||
self.state != "succeeded" or self.result_id is None or self.result_sha256 is None
|
||||
):
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"recorded-job publication lifecycle has no execution result"
|
||||
@@ -854,10 +848,7 @@ class ObservatoryRecordedReconciliationRequest:
|
||||
_validate_text(self.reason, "reconciliation reason", max_length=1_000)
|
||||
if self.resource_release_attestation.job_id != self.job_id:
|
||||
raise ValueError("resource-release attestation is bound to another job")
|
||||
if (
|
||||
self.resource_release_attestation.claim_generation
|
||||
!= self.expected_claim_generation
|
||||
):
|
||||
if self.resource_release_attestation.claim_generation != self.expected_claim_generation:
|
||||
raise ValueError("resource-release attestation is bound to another generation")
|
||||
|
||||
@property
|
||||
@@ -1237,16 +1228,15 @@ class ObservatoryRecordedJobQueue:
|
||||
if reject_duplicate_computation:
|
||||
# This check shares the INSERT transaction: two clients with
|
||||
# different idempotency keys cannot race into duplicate jobs.
|
||||
# A sealed result awaiting publication is not a reason to run
|
||||
# the models again. Published-cache validity is a separate gate.
|
||||
# One sealed recording/profile version is calculated once after
|
||||
# it has started successfully. Failed attempts remain as audit
|
||||
# evidence and may be resubmitted as a fresh queue attempt.
|
||||
duplicate = connection.execute(
|
||||
"SELECT job_id FROM observatory_recorded_jobs "
|
||||
"WHERE identity_sha256 = ? AND ("
|
||||
"state IN ('accepted', 'queued', 'claimed', 'running', 'paused', "
|
||||
"'preemption-pending', 'reconciliation-required') OR "
|
||||
"(state = 'succeeded' AND publication_state IN ('pending', 'failed'))) "
|
||||
"WHERE source_session_id = ? AND setup_id = ? "
|
||||
"AND definition_sha256 = ? AND state != 'failed' "
|
||||
"ORDER BY created_at_utc DESC, job_id DESC LIMIT 1",
|
||||
(identity_sha256,),
|
||||
(intent.source_session_id, definition.setup_id, definition.definition_sha256),
|
||||
).fetchone()
|
||||
if duplicate is not None:
|
||||
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
|
||||
@@ -1361,12 +1351,13 @@ class ObservatoryRecordedJobQueue:
|
||||
grant_count = connection.execute(
|
||||
"SELECT COUNT(*) FROM observatory_recorded_claim_grants_v3"
|
||||
).fetchone()[0]
|
||||
job_counts = dict(connection.execute(
|
||||
"SELECT state, COUNT(*) FROM observatory_recorded_jobs GROUP BY state"
|
||||
).fetchall())
|
||||
job_counts = dict(
|
||||
connection.execute(
|
||||
"SELECT state, COUNT(*) FROM observatory_recorded_jobs GROUP BY state"
|
||||
).fetchall()
|
||||
)
|
||||
live_count = connection.execute(
|
||||
"SELECT COUNT(*) FROM observatory_live_leases "
|
||||
"WHERE state IN ('pending', 'active')"
|
||||
"SELECT COUNT(*) FROM observatory_live_leases WHERE state IN ('pending', 'active')"
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-claim-readiness/v1",
|
||||
@@ -1622,7 +1613,11 @@ class ObservatoryRecordedJobQueue:
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def report_progress(
|
||||
self, job_id: str, *, claim_token: str, claimant_id: str,
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
claim_token: str,
|
||||
claimant_id: str,
|
||||
progress: RecordedProgress,
|
||||
) -> None:
|
||||
"""Replace one small observation, fenced in the ownership transaction."""
|
||||
@@ -1665,10 +1660,11 @@ class ObservatoryRecordedJobQueue:
|
||||
job = self._get_job(connection, job_id)
|
||||
row = connection.execute(
|
||||
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
|
||||
"WHERE job_id = ?", (job_id,),
|
||||
"WHERE job_id = ?",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
progress = None if row is None else RecordedProgress.model_validate_json(
|
||||
row["snapshot_json"]
|
||||
progress = (
|
||||
None if row is None else RecordedProgress.model_validate_json(row["snapshot_json"])
|
||||
)
|
||||
if progress is not None and progress.claim_generation != job.claim_generation:
|
||||
progress = None
|
||||
@@ -1681,10 +1677,15 @@ class ObservatoryRecordedJobQueue:
|
||||
"claim_generation": job.claim_generation,
|
||||
"state": job.state,
|
||||
"received_at_utc": None if progress is None else row["received_at_utc"],
|
||||
"age_seconds": None if progress is None else max(0.0, (
|
||||
_parse_timestamp(self._timestamp(), "clock")
|
||||
- _parse_timestamp(row["received_at_utc"], "progress receipt")
|
||||
).total_seconds()),
|
||||
"age_seconds": None
|
||||
if progress is None
|
||||
else max(
|
||||
0.0,
|
||||
(
|
||||
_parse_timestamp(self._timestamp(), "clock")
|
||||
- _parse_timestamp(row["received_at_utc"], "progress receipt")
|
||||
).total_seconds(),
|
||||
),
|
||||
"progress": None if progress is None else progress.model_dump(mode="json"),
|
||||
}
|
||||
|
||||
@@ -1720,10 +1721,7 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
self._require_active_claim(job, claim_token, now=self._timestamp())
|
||||
if (
|
||||
job.claim_generation != claim_generation
|
||||
or job.active_claimant_id != claimant_id
|
||||
):
|
||||
if job.claim_generation != claim_generation or job.active_claimant_id != claimant_id:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim ownership is stale"
|
||||
)
|
||||
@@ -1756,8 +1754,7 @@ class ObservatoryRecordedJobQueue:
|
||||
|
||||
with self._transaction() as connection:
|
||||
existing = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_reconciliations "
|
||||
"WHERE reconciliation_id = ?",
|
||||
"SELECT * FROM observatory_recorded_reconciliations WHERE reconciliation_id = ?",
|
||||
(request.reconciliation_id,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
@@ -2060,9 +2057,7 @@ class ObservatoryRecordedJobQueue:
|
||||
with self._transaction() as connection:
|
||||
job = self._get_job(connection, job_id)
|
||||
if job.state != "succeeded":
|
||||
raise ObservatoryRecordedQueueConflictError(
|
||||
"recorded execution has not succeeded"
|
||||
)
|
||||
raise ObservatoryRecordedQueueConflictError("recorded execution has not succeeded")
|
||||
if job.publication_state == "published":
|
||||
return job
|
||||
if job.publication_state not in {"pending", "failed"}:
|
||||
@@ -2080,15 +2075,39 @@ class ObservatoryRecordedJobQueue:
|
||||
)
|
||||
return self._get_job(connection, job_id)
|
||||
|
||||
def pending_publications(self) -> tuple[ObservatoryRecordedJob, ...]:
|
||||
"""Return durable outbox entries in deterministic retry order."""
|
||||
def pending_publications(
|
||||
self,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
after: tuple[str, str] | None = None,
|
||||
) -> tuple[ObservatoryRecordedJob, ...]:
|
||||
"""Read an outbox page without retaining the entire failed-job history.
|
||||
|
||||
The cursor uses immutable creation identity, not ``updated_at_utc``:
|
||||
attempts and concurrent publication cannot move entries across pages.
|
||||
The no-argument form preserves the existing internal inspection API.
|
||||
"""
|
||||
if limit is not None and (type(limit) is not int or not 1 <= limit <= 100):
|
||||
raise ValueError("publication page limit must be within 1..100")
|
||||
conditions = ["publication_state IN ('pending', 'failed')"]
|
||||
parameters: list[object] = []
|
||||
if after is not None:
|
||||
if not isinstance(after, tuple) or len(after) != 2:
|
||||
raise ValueError("publication cursor is invalid")
|
||||
_validate_timestamp(after[0], "publication cursor timestamp")
|
||||
_validate_pattern(after[1], _JOB_ID, "publication cursor job id")
|
||||
conditions.append("(created_at_utc, job_id) > (?, ?)")
|
||||
parameters.extend(after)
|
||||
query = (
|
||||
"SELECT * FROM observatory_recorded_jobs WHERE "
|
||||
+ " AND ".join(conditions)
|
||||
+ " ORDER BY created_at_utc, job_id"
|
||||
)
|
||||
if limit is not None:
|
||||
query += " LIMIT ?"
|
||||
parameters.append(limit)
|
||||
with self._read_connection() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observatory_recorded_jobs "
|
||||
"WHERE publication_state IN ('pending', 'failed') "
|
||||
"ORDER BY created_at_utc, job_id"
|
||||
).fetchall()
|
||||
rows = connection.execute(query, parameters).fetchall()
|
||||
return tuple(_job_from_row(row) for row in rows)
|
||||
|
||||
def fail(
|
||||
@@ -2187,8 +2206,12 @@ class ObservatoryRecordedJobQueue:
|
||||
return tuple(_job_from_row(row) for row in rows)
|
||||
|
||||
def published_results(
|
||||
self, *, source_session_id: str, source_catalog_sha256: str,
|
||||
setup_id: str, definition_sha256: str,
|
||||
self,
|
||||
*,
|
||||
source_session_id: str,
|
||||
source_catalog_sha256: str,
|
||||
setup_id: str,
|
||||
definition_sha256: str,
|
||||
) -> tuple[ObservatoryRecordedJob, ...]:
|
||||
"""Exact cache candidates, never inferred from labels or a truncated job page."""
|
||||
|
||||
@@ -2527,11 +2550,10 @@ class ObservatoryRecordedJobQueue:
|
||||
)
|
||||
if exact_replay:
|
||||
return job
|
||||
if (
|
||||
job.terminal_claim_token_sha256 != token_sha256
|
||||
or job.terminal_code
|
||||
in {"claim-lease-expired", "claim-lease-migration"}
|
||||
):
|
||||
if job.terminal_claim_token_sha256 != token_sha256 or job.terminal_code in {
|
||||
"claim-lease-expired",
|
||||
"claim-lease-migration",
|
||||
}:
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job terminal acknowledgement is stale"
|
||||
)
|
||||
@@ -2618,9 +2640,7 @@ class ObservatoryRecordedJobQueue:
|
||||
or _parse_timestamp(now, "queue timestamp")
|
||||
>= _parse_timestamp(job.claim_expires_at_utc, "claim expiry")
|
||||
):
|
||||
raise ObservatoryRecordedQueueStaleClaimError(
|
||||
"recorded-job claim lease expired"
|
||||
)
|
||||
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim lease expired")
|
||||
|
||||
def _recover_stale_claims(
|
||||
self,
|
||||
@@ -3162,10 +3182,7 @@ def _reconciliation_receipt_from_row(
|
||||
evidence_sha256=row["resource_release_evidence_sha256"],
|
||||
attested_at_utc=row["resource_release_attested_at_utc"],
|
||||
)
|
||||
if (
|
||||
attestation.attestation_sha256
|
||||
!= row["resource_release_attestation_sha256"]
|
||||
):
|
||||
if attestation.attestation_sha256 != row["resource_release_attestation_sha256"]:
|
||||
raise ObservatoryRecordedQueueIntegrityError(
|
||||
"stored resource-release attestation identity changed"
|
||||
)
|
||||
@@ -3433,9 +3450,7 @@ def _validate_claim_lease_seconds(value: object) -> None:
|
||||
if (
|
||||
not isinstance(value, int)
|
||||
or isinstance(value, bool)
|
||||
or not MIN_RECORDED_CLAIM_LEASE_SECONDS
|
||||
<= value
|
||||
<= MAX_RECORDED_CLAIM_LEASE_SECONDS
|
||||
or not MIN_RECORDED_CLAIM_LEASE_SECONDS <= value <= MAX_RECORDED_CLAIM_LEASE_SECONDS
|
||||
):
|
||||
raise ValueError("recorded-job claim lease duration is invalid")
|
||||
|
||||
|
||||
@@ -340,6 +340,20 @@ class RecordedK1SourceAdmissionService:
|
||||
prepare_media=False,
|
||||
)
|
||||
|
||||
def prepare_check(self, source_session_id: str) -> PortableRecordedSourceAdmission:
|
||||
"""Prepare the recorded-media manifest and return a non-persistent check.
|
||||
|
||||
This is the bounded one-click path used when a compatible recording has
|
||||
not previously been opened in the viewer. Source bundle documents are
|
||||
still written only by ``admit`` after the returned identity is fenced.
|
||||
"""
|
||||
|
||||
return self._prepare(
|
||||
source_session_id,
|
||||
persist=False,
|
||||
prepare_media=True,
|
||||
)
|
||||
|
||||
def admit(
|
||||
self,
|
||||
source_session_id: str,
|
||||
@@ -606,14 +620,10 @@ class RecordedK1SourceAdmissionService:
|
||||
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
|
||||
exact_metadata_member = (
|
||||
catalog_artifact is not None
|
||||
and replay_artifact.artifact_id
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and catalog_artifact.kind
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and replay_artifact.media_type
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
and catalog_artifact.media_type
|
||||
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
and replay_artifact.artifact_id == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and catalog_artifact.kind == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
and replay_artifact.media_type == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
and catalog_artifact.media_type == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
)
|
||||
digest_matches_catalog = (
|
||||
replay_artifact.expected_sha256 == catalog_artifact.sha256
|
||||
@@ -854,13 +864,10 @@ def _seal_replay_artifact_digests(
|
||||
catalog_artifact = catalog_artifacts.get(artifact.artifact_id)
|
||||
if (
|
||||
catalog_artifact is None
|
||||
or artifact.artifact_id
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or catalog_artifact.kind
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or artifact.artifact_id != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or catalog_artifact.kind != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
|
||||
or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
or catalog_artifact.media_type
|
||||
!= PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
or catalog_artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
|
||||
or catalog_artifact.sha256 is not None
|
||||
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
|
||||
or artifact.file_byte_length != catalog_artifact.byte_length
|
||||
|
||||
@@ -15,8 +15,10 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
@@ -36,6 +38,7 @@ from k1link.observatory.recorded_progress import observe_recorded_execution
|
||||
|
||||
WORKER_006_CONTOUR_ID: Final = "worker-006"
|
||||
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
_JOB_ID_PATTERN: Final = r"^observatory-run-[a-f0-9]{32}$"
|
||||
_CLAIM_TOKEN_PATTERN: Final = r"^[a-f0-9]{64}$"
|
||||
@@ -97,6 +100,10 @@ class ObservatoryWorkerClaimRejectedError(ObservatoryWorkerAgentError):
|
||||
"""A transport supplied an unknown, spoofed, or corrupted claim."""
|
||||
|
||||
|
||||
class ObservatoryWorkerTransientTransportError(ObservatoryWorkerAgentError):
|
||||
"""A bounded request may be replayed with its unchanged idempotent identity."""
|
||||
|
||||
|
||||
class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError):
|
||||
"""No local adapter matches the exact sealed executor identity."""
|
||||
|
||||
@@ -505,16 +512,28 @@ class ObservatoryWorkerAgent:
|
||||
observe_recorded_execution(
|
||||
active_job.claim_generation,
|
||||
lambda snapshot: send_progress(
|
||||
job_id=active_job.job_id, claim_token=claim.claim_token,
|
||||
job_id=active_job.job_id,
|
||||
claim_token=claim.claim_token,
|
||||
progress=snapshot,
|
||||
),
|
||||
) if callable(send_progress) else nullcontext()
|
||||
)
|
||||
if callable(send_progress)
|
||||
else nullcontext()
|
||||
)
|
||||
with observation:
|
||||
result = adapter.execute(active_job)
|
||||
if not isinstance(result, ObservatoryWorkerExecutionResult):
|
||||
raise TypeError("executor returned an unknown result contract")
|
||||
except Exception as exc:
|
||||
# Do not expose exception text, request headers, paths or secrets.
|
||||
# Preserve the independent executor failure even if its claim was lost.
|
||||
_LOG.error(
|
||||
"Recorded executor failed job=%s generation=%s error_class=%s cause_class=%s",
|
||||
active_job.job_id,
|
||||
active_job.claim_generation,
|
||||
type(exc).__name__,
|
||||
type(exc.__cause__).__name__ if exc.__cause__ else "none",
|
||||
)
|
||||
heartbeat.stop()
|
||||
if heartbeat.failed:
|
||||
return ObservatoryWorkerCycleReport(
|
||||
@@ -594,6 +613,7 @@ class _ClaimHeartbeat:
|
||||
claim_token: str,
|
||||
interval_seconds: float,
|
||||
stop_timeout_seconds: float,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("Worker heartbeat interval must be positive")
|
||||
@@ -602,6 +622,11 @@ class _ClaimHeartbeat:
|
||||
self._claim_token = claim_token
|
||||
self._interval_seconds = interval_seconds
|
||||
self._stop_timeout_seconds = stop_timeout_seconds
|
||||
self._clock = clock
|
||||
self._lease_seconds = _heartbeat_lease_seconds(
|
||||
job.claim_expires_at_utc,
|
||||
job.claim_heartbeat_at_utc,
|
||||
)
|
||||
self._stop = threading.Event()
|
||||
self._state_lock = threading.Lock()
|
||||
self._failure: Exception | None = None
|
||||
@@ -631,11 +656,20 @@ class _ClaimHeartbeat:
|
||||
|
||||
def _run(self) -> None:
|
||||
sequence = self._job.claim_renewal_count + 1
|
||||
# Local monotonic deadlines do not depend on Worker/Core clock skew.
|
||||
# Server-side lease/generation checks remain the final authority.
|
||||
deadline = self._clock() + self._lease_seconds
|
||||
sequence_started = self._clock()
|
||||
retrying = False
|
||||
# Renew immediately once start has been acknowledged. Waiting a full
|
||||
# interval here would assume that claim/start transport latency consumed
|
||||
# none of the original lease window.
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
if self._clock() >= deadline:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker heartbeat lease budget expired"
|
||||
)
|
||||
acknowledgement = self._transport.renew_claim(
|
||||
claimant_id=WORKER_006_CONTOUR_ID,
|
||||
job_id=self._job.job_id,
|
||||
@@ -648,25 +682,74 @@ class _ClaimHeartbeat:
|
||||
expected_job=self._job,
|
||||
expected_state=("claimed", "running"),
|
||||
)
|
||||
if (
|
||||
renewed.claim_lease is None
|
||||
or renewed.claim_lease.renewal_count != sequence
|
||||
):
|
||||
if renewed.claim_lease is None or renewed.claim_lease.renewal_count != sequence:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker heartbeat acknowledgement changed its sequence"
|
||||
)
|
||||
deadline = sequence_started + _heartbeat_lease_seconds(
|
||||
renewed.claim_lease.expires_at_utc,
|
||||
renewed.claim_lease.heartbeat_at_utc,
|
||||
)
|
||||
if self._clock() >= deadline:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker heartbeat acknowledgement expired"
|
||||
)
|
||||
if retrying:
|
||||
_LOG.warning(
|
||||
"Recorded heartbeat recovered job=%s generation=%s sequence=%s",
|
||||
self._job.job_id,
|
||||
self._job.claim_generation,
|
||||
sequence,
|
||||
)
|
||||
retrying = False
|
||||
sequence += 1
|
||||
except ObservatoryWorkerTransientTransportError as exc:
|
||||
if not retrying:
|
||||
_LOG.warning(
|
||||
"Recorded heartbeat retry job=%s generation=%s sequence=%s error_class=%s",
|
||||
self._job.job_id,
|
||||
self._job.claim_generation,
|
||||
sequence,
|
||||
type(exc).__name__,
|
||||
)
|
||||
retrying = True
|
||||
remaining = deadline - self._clock()
|
||||
if remaining <= 0 or self._stop.wait(
|
||||
min(5.0, self._interval_seconds, max(0.0, remaining))
|
||||
):
|
||||
# Finishing compute during an unacknowledged renewal does
|
||||
# not turn uncertain ownership into successful publication.
|
||||
self._record_failure(exc)
|
||||
return
|
||||
continue
|
||||
except Exception as exc:
|
||||
self._record_failure(exc)
|
||||
self._stop.set()
|
||||
return
|
||||
if self._stop.wait(self._interval_seconds):
|
||||
return
|
||||
sequence_started = self._clock()
|
||||
|
||||
def _record_failure(self, exc: Exception) -> None:
|
||||
with self._state_lock:
|
||||
if self._failure is None:
|
||||
self._failure = exc
|
||||
_LOG.error(
|
||||
"Recorded heartbeat lost job=%s generation=%s error_class=%s cause_class=%s",
|
||||
self._job.job_id,
|
||||
self._job.claim_generation,
|
||||
type(exc).__name__,
|
||||
type(exc.__cause__).__name__ if exc.__cause__ else "none",
|
||||
)
|
||||
|
||||
|
||||
def _heartbeat_lease_seconds(expires: str | None, renewed: str | None) -> float:
|
||||
if expires is None or renewed is None:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker heartbeat has no lease clock")
|
||||
duration = (datetime.fromisoformat(expires) - datetime.fromisoformat(renewed)).total_seconds()
|
||||
if not 0 < duration <= 3600:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker heartbeat lease duration is invalid")
|
||||
return duration
|
||||
|
||||
|
||||
def _validate_claim(
|
||||
@@ -687,8 +770,7 @@ def _validate_claim(
|
||||
"claim_request_id": claim_request_id,
|
||||
"claimant_id": WORKER_006_CONTOUR_ID,
|
||||
"supported_executor_identities": [
|
||||
identity.as_dict()
|
||||
for identity in sorted(supported_executor_identities)
|
||||
identity.as_dict() for identity in sorted(supported_executor_identities)
|
||||
],
|
||||
}
|
||||
)
|
||||
@@ -699,9 +781,7 @@ def _validate_claim(
|
||||
"Worker claim job is not in a claimed generation"
|
||||
)
|
||||
if claim.job.claim_lease is None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim has no renewable lease"
|
||||
)
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim has no renewable lease")
|
||||
if claim.job.result is not None or claim.job.terminal is not None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim already carries a terminal outcome"
|
||||
@@ -889,12 +969,9 @@ def _default_claim_request_id() -> str:
|
||||
|
||||
def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float:
|
||||
if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None:
|
||||
raise ObservatoryWorkerClaimRejectedError(
|
||||
"Worker claim has no heartbeat lease bounds"
|
||||
)
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim has no heartbeat lease bounds")
|
||||
remaining = (
|
||||
_parse_timestamp(job.claim_expires_at_utc)
|
||||
- _parse_timestamp(job.claim_heartbeat_at_utc)
|
||||
_parse_timestamp(job.claim_expires_at_utc) - _parse_timestamp(job.claim_heartbeat_at_utc)
|
||||
).total_seconds()
|
||||
if remaining <= 0:
|
||||
raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired")
|
||||
|
||||
@@ -56,9 +56,11 @@ from k1link.observatory.worker_agent import (
|
||||
WORKER_006_CONTOUR_ID,
|
||||
ObservatoryWorkerExecutionResult,
|
||||
ObservatoryWorkerExecutorIdentity,
|
||||
ObservatoryWorkerTransientTransportError,
|
||||
ObservatoryWorkerTransport,
|
||||
SealedObservatoryRecordedJob,
|
||||
)
|
||||
from k1link.observatory.worker_source_cache import WorkerSourceCache, WorkerSourceCacheError
|
||||
|
||||
WORKER_HTTP_MAX_JSON_BYTES: Final = 16 * 1024 * 1024
|
||||
WORKER_HTTP_COPY_CHUNK_BYTES: Final = 1024 * 1024
|
||||
@@ -84,6 +86,13 @@ class ObservatoryWorkerHttpError(RuntimeError):
|
||||
"""The authenticated Worker HTTP boundary failed closed."""
|
||||
|
||||
|
||||
class ObservatoryWorkerHttpTransientError(
|
||||
ObservatoryWorkerHttpError,
|
||||
ObservatoryWorkerTransientTransportError,
|
||||
):
|
||||
"""Temporary unavailability, not permission to replay arbitrary operations."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ClaimContext:
|
||||
job_id: str
|
||||
@@ -128,9 +137,7 @@ class _SourceDestinationLayout:
|
||||
if member.primary:
|
||||
return self.root / "mqtt.raw.k1mqtt"
|
||||
if self.spatial is None:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"spatial source destination is unavailable"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("spatial source destination is unavailable")
|
||||
return self.spatial / member.member_id
|
||||
if member.kind == "spatial-replay-metadata":
|
||||
return self.root / "mqtt.metadata.jsonl"
|
||||
@@ -165,6 +172,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
contour_id: str = WORKER_006_CONTOUR_ID,
|
||||
timeout_seconds: float = WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
source_cache_root: Path | None = None,
|
||||
) -> None:
|
||||
self._base_url = _validated_base_url(base_url)
|
||||
if _TOKEN.fullmatch(bearer_token) is None:
|
||||
@@ -180,6 +188,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
}
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._work_root = _secure_directory(work_root)
|
||||
self._source_cache = WorkerSourceCache(
|
||||
source_cache_root or self._work_root / "source-cas-v1"
|
||||
)
|
||||
self._client = httpx.Client(
|
||||
base_url=self._base_url,
|
||||
headers=self._headers,
|
||||
@@ -217,8 +228,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"schema_version": "missioncore.observatory-worker-claim-request/v3",
|
||||
"claim_request_id": claim_request_id,
|
||||
"supported_executor_identities": [
|
||||
identity.as_dict()
|
||||
for identity in sorted(supported_executor_identities)
|
||||
identity.as_dict() for identity in sorted(supported_executor_identities)
|
||||
],
|
||||
},
|
||||
allow_empty=True,
|
||||
@@ -272,6 +282,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
return self._required_json_request(
|
||||
"POST",
|
||||
self._job_path(job_id, "lease/renew"),
|
||||
timeout_seconds=5.0,
|
||||
json_body={
|
||||
"schema_version": "missioncore.observatory-worker-renew-request/v1",
|
||||
"claim_token": claim_token,
|
||||
@@ -281,18 +292,25 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
)
|
||||
|
||||
def report_progress(
|
||||
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
claim_token: str,
|
||||
progress: RecordedProgress,
|
||||
) -> None:
|
||||
context = self._require_cached_claim(job_id, claim_token)
|
||||
if progress.claim_generation != context.claim_generation:
|
||||
raise ObservatoryWorkerHttpError("progress generation changed")
|
||||
# No response body is needed. Bounded I/O cannot stall the execution thread.
|
||||
with self._client.stream(
|
||||
"POST", self._job_path(job_id, "progress"),
|
||||
"POST",
|
||||
self._job_path(job_id, "progress"),
|
||||
json={
|
||||
"schema_version": "missioncore.observatory-worker-progress-request/v1",
|
||||
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
|
||||
}, timeout=httpx.Timeout(2.0),
|
||||
"claim_token": claim_token,
|
||||
"progress": progress.model_dump(mode="json"),
|
||||
},
|
||||
timeout=httpx.Timeout(2.0),
|
||||
) as response:
|
||||
if response.status_code != 204:
|
||||
raise ObservatoryWorkerHttpError("progress observation was not accepted")
|
||||
@@ -354,7 +372,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
headers=headers,
|
||||
)
|
||||
members = _source_members(manifest, job)
|
||||
report_recorded_progress("source-transfer", 0, len(members), "members")
|
||||
report_recorded_progress("source-preparation", 0, len(members), "members")
|
||||
root = _secure_directory(
|
||||
self._work_root
|
||||
/ "sources"
|
||||
@@ -371,40 +389,56 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"source materialization members select the same local role"
|
||||
)
|
||||
destinations[destination] = member
|
||||
ready_members: set[str] = set()
|
||||
try:
|
||||
for checked_members, (destination, member) in enumerate(destinations.items(), start=1):
|
||||
if _matches_file(
|
||||
destination, member.sha256, member.byte_length
|
||||
) or self._source_cache.restore(
|
||||
destination,
|
||||
sha256=member.sha256,
|
||||
byte_length=member.byte_length,
|
||||
):
|
||||
ready_members.add(member.member_id)
|
||||
report_recorded_progress(
|
||||
"source-preparation", checked_members, len(members), "members"
|
||||
)
|
||||
except WorkerSourceCacheError as exc:
|
||||
raise ObservatoryWorkerHttpError("Worker source cache admission failed") from exc
|
||||
camera_members = _ordered_camera_epoch_members(members)
|
||||
camera_epoch_ready = all(
|
||||
_matches_file(
|
||||
layout.destination(member),
|
||||
member.sha256,
|
||||
member.byte_length,
|
||||
)
|
||||
for member in camera_members
|
||||
)
|
||||
if not camera_epoch_ready:
|
||||
camera_epoch_ready = all(member.member_id in ready_members for member in camera_members)
|
||||
# A partly cached epoch needs only missing members, not another complete
|
||||
# epoch archive. Cold delivery retains the existing packed transport.
|
||||
if not camera_epoch_ready and not any(
|
||||
member.member_id in ready_members for member in camera_members
|
||||
):
|
||||
camera_epoch_ready = self._download_camera_epoch_archive(
|
||||
job=job,
|
||||
context=context,
|
||||
members=camera_members,
|
||||
layout=layout,
|
||||
)
|
||||
completed_members = len(camera_members) if camera_epoch_ready else 0
|
||||
if camera_epoch_ready:
|
||||
ready_members.update(member.member_id for member in camera_members)
|
||||
completed_members = len(ready_members)
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
for destination, member in destinations.items():
|
||||
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
|
||||
continue
|
||||
if _matches_file(destination, member.sha256, member.byte_length):
|
||||
completed_members += 1
|
||||
report_recorded_progress(
|
||||
"source-transfer", completed_members, len(members), "members",
|
||||
if member.member_id not in ready_members:
|
||||
self._download_member(
|
||||
job_id=job.job_id,
|
||||
context=context,
|
||||
member=member,
|
||||
destination=destination,
|
||||
)
|
||||
continue
|
||||
self._download_member(
|
||||
job_id=job.job_id,
|
||||
context=context,
|
||||
member=member,
|
||||
destination=destination,
|
||||
)
|
||||
completed_members += 1
|
||||
completed_members += 1
|
||||
try:
|
||||
self._source_cache.retain(
|
||||
destination,
|
||||
sha256=member.sha256,
|
||||
byte_length=member.byte_length,
|
||||
)
|
||||
except WorkerSourceCacheError as exc:
|
||||
raise ObservatoryWorkerHttpError("Worker source cache publication failed") from exc
|
||||
report_recorded_progress("source-transfer", completed_members, len(members), "members")
|
||||
manifest_path = root / "materialization-manifest.json"
|
||||
_write_local_exact(manifest_path, canonical_json(manifest))
|
||||
@@ -426,9 +460,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
try:
|
||||
package = PortableResultPackageManifest.from_bytes(manifest_payload)
|
||||
except Exception as exc:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"Worker result draft manifest is invalid"
|
||||
) from exc
|
||||
raise ObservatoryWorkerHttpError("Worker result draft manifest is invalid") from exc
|
||||
if (
|
||||
package.manifest_sha256 != draft.result_sha256
|
||||
or package.result.get("result_id") != draft.result_id
|
||||
@@ -441,9 +473,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"claim_generation": job.claim_generation,
|
||||
}
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"Worker result draft belongs to another sealed job"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("Worker result draft belongs to another sealed job")
|
||||
headers = self._claim_headers(context)
|
||||
path = self._job_path(
|
||||
job.job_id,
|
||||
@@ -456,9 +486,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
content=manifest_payload,
|
||||
)
|
||||
if plan.get("package_identity_sha256") != package.identity_sha256:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"result upload plan uses another package identity"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("result upload plan uses another package identity")
|
||||
upload_members = _upload_members(plan, job, draft)
|
||||
artifacts = {artifact.role: artifact for artifact in package.artifacts}
|
||||
if (
|
||||
@@ -478,19 +506,25 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
}
|
||||
)
|
||||
).hexdigest()
|
||||
if artifact is None or (
|
||||
artifact.media_type,
|
||||
artifact.byte_length,
|
||||
artifact.sha256,
|
||||
) != (member.media_type, member.byte_length, member.sha256) or (
|
||||
member.member_id != expected_member_id
|
||||
if (
|
||||
artifact is None
|
||||
or (
|
||||
artifact.media_type,
|
||||
artifact.byte_length,
|
||||
artifact.sha256,
|
||||
)
|
||||
!= (member.media_type, member.byte_length, member.sha256)
|
||||
or (member.member_id != expected_member_id)
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"result upload plan differs from the local manifest"
|
||||
)
|
||||
if member.uploaded:
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
"result-transfer",
|
||||
member_index + 1,
|
||||
len(upload_members),
|
||||
"members",
|
||||
)
|
||||
continue
|
||||
relative = relative_artifact_path(artifact.relative_path)
|
||||
@@ -500,10 +534,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"PUT",
|
||||
self._job_path(
|
||||
job.job_id,
|
||||
(
|
||||
f"result-packages/{draft.result_sha256}/members/"
|
||||
f"{member.member_id}"
|
||||
),
|
||||
(f"result-packages/{draft.result_sha256}/members/{member.member_id}"),
|
||||
),
|
||||
headers={**headers, "Content-Type": member.media_type},
|
||||
content=stream,
|
||||
@@ -514,14 +545,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
)
|
||||
updated_members = _upload_members(updated, job, draft)
|
||||
if not any(
|
||||
item.member_id == member.member_id and item.uploaded
|
||||
for item in updated_members
|
||||
item.member_id == member.member_id and item.uploaded for item in updated_members
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"result upload acknowledgement did not seal its member"
|
||||
)
|
||||
report_recorded_progress(
|
||||
"result-transfer", member_index + 1, len(upload_members), "members",
|
||||
"result-transfer",
|
||||
member_index + 1,
|
||||
len(upload_members),
|
||||
"members",
|
||||
)
|
||||
receipt = self._required_json_request(
|
||||
"POST",
|
||||
@@ -536,23 +569,17 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
"job_id": job.job_id,
|
||||
"job_identity_sha256": job.identity_sha256,
|
||||
"claim_generation": job.claim_generation,
|
||||
"claim_token_sha256": hashlib.sha256(
|
||||
context.claim_token.encode("ascii")
|
||||
).hexdigest(),
|
||||
"claim_token_sha256": hashlib.sha256(context.claim_token.encode("ascii")).hexdigest(),
|
||||
"result_id": draft.result_id,
|
||||
"result_sha256": draft.result_sha256,
|
||||
"package_identity_sha256": package.identity_sha256,
|
||||
"member_count": len(package.artifacts),
|
||||
"total_bytes": sum(
|
||||
artifact.byte_length for artifact in package.artifacts
|
||||
),
|
||||
"total_bytes": sum(artifact.byte_length for artifact in package.artifacts),
|
||||
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
|
||||
}
|
||||
expected_receipt = {
|
||||
**receipt_identity,
|
||||
"receipt_sha256": hashlib.sha256(
|
||||
canonical_json(receipt_identity)
|
||||
).hexdigest(),
|
||||
"receipt_sha256": hashlib.sha256(canonical_json(receipt_identity)).hexdigest(),
|
||||
}
|
||||
if receipt != expected_receipt:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
@@ -593,20 +620,12 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
return False
|
||||
self._raise_for_status(response)
|
||||
content_encoding = response.headers.get("content-encoding")
|
||||
if (
|
||||
content_encoding is not None
|
||||
and content_encoding.lower() != "identity"
|
||||
):
|
||||
if content_encoding is not None and content_encoding.lower() != "identity":
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive content encoding changed"
|
||||
)
|
||||
if (
|
||||
response.headers.get("content-type")
|
||||
!= PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive media type changed"
|
||||
)
|
||||
if response.headers.get("content-type") != PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE:
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive media type changed")
|
||||
declared = response.headers.get("content-length")
|
||||
if (
|
||||
declared is None
|
||||
@@ -615,9 +634,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
or len(declared) > len(str(MAX_CAMERA_EPOCH_ARCHIVE_BYTES))
|
||||
or str(int(declared)) != declared
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive length is invalid"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive length is invalid")
|
||||
expected_bytes = int(declared)
|
||||
expected_sha256 = response.headers.get(_CONTENT_SHA_HEADER, "")
|
||||
archive_id = response.headers.get(
|
||||
@@ -629,15 +646,10 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
or _SHA256.fullmatch(expected_sha256) is None
|
||||
or _SHA256.fullmatch(archive_id) is None
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive seal is invalid"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive seal is invalid")
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY
|
||||
| os.O_CREAT
|
||||
| os.O_EXCL
|
||||
| getattr(os, "O_NOFOLLOW", 0),
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
@@ -671,9 +683,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
sha256=expected_sha256,
|
||||
)
|
||||
if archive_id != expected_archive_id:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive identity changed"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive identity changed")
|
||||
_extract_camera_epoch_archive(
|
||||
temporary,
|
||||
layout=layout,
|
||||
@@ -715,28 +725,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
) as response:
|
||||
self._raise_for_status(response)
|
||||
content_encoding = response.headers.get("content-encoding")
|
||||
if (
|
||||
content_encoding is not None
|
||||
and content_encoding.lower() != "identity"
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source member content encoding changed"
|
||||
)
|
||||
if content_encoding is not None and content_encoding.lower() != "identity":
|
||||
raise ObservatoryWorkerHttpError("source member content encoding changed")
|
||||
declared = response.headers.get("content-length")
|
||||
if declared is not None and declared != str(member.byte_length):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source member Content-Length changed"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source member Content-Length changed")
|
||||
if response.headers.get(_CONTENT_SHA_HEADER) != member.sha256:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source member digest header changed"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source member digest header changed")
|
||||
descriptor = os.open(
|
||||
temporary,
|
||||
os.O_WRONLY
|
||||
| os.O_CREAT
|
||||
| os.O_EXCL
|
||||
| getattr(os, "O_NOFOLLOW", 0),
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
@@ -751,9 +749,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
if byte_length != member.byte_length or digest.hexdigest() != member.sha256:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source member content differs from its manifest"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source member content differs from its manifest")
|
||||
_publish_local_file(temporary, destination, member.sha256, member.byte_length)
|
||||
finally:
|
||||
with suppress(FileNotFoundError):
|
||||
@@ -767,6 +763,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
json_body: object | None = None,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
content: bytes | object | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> dict[str, object]:
|
||||
payload = self._json_request(
|
||||
method,
|
||||
@@ -774,6 +771,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
json_body=json_body,
|
||||
headers=headers,
|
||||
content=content,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
if payload is None:
|
||||
raise ObservatoryWorkerHttpError("Worker endpoint returned no document")
|
||||
@@ -788,6 +786,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
headers: Mapping[str, str] | None = None,
|
||||
content: bytes | object | None = None,
|
||||
allow_empty: bool = False,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> dict[str, object] | None:
|
||||
try:
|
||||
with self._client.stream(
|
||||
@@ -796,6 +795,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
json=json_body,
|
||||
headers=headers,
|
||||
content=content, # type: ignore[arg-type]
|
||||
timeout=self._client.timeout if timeout_seconds is None else timeout_seconds,
|
||||
) as response:
|
||||
if allow_empty and response.status_code == 204:
|
||||
return None
|
||||
@@ -804,11 +804,13 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
for chunk in response.iter_bytes():
|
||||
payload.extend(chunk)
|
||||
if len(payload) > WORKER_HTTP_MAX_JSON_BYTES:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"Worker JSON response exceeds bounds"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("Worker JSON response exceeds bounds")
|
||||
except ObservatoryWorkerHttpError:
|
||||
raise
|
||||
except httpx.RequestError as exc:
|
||||
raise ObservatoryWorkerHttpTransientError(
|
||||
"Worker HTTP transport is unavailable"
|
||||
) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise ObservatoryWorkerHttpError("Worker HTTP transport is unavailable") from exc
|
||||
if not payload:
|
||||
@@ -825,6 +827,10 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
|
||||
if response.is_redirect:
|
||||
raise ObservatoryWorkerHttpError("Worker endpoint redirect was rejected")
|
||||
if response.status_code < 200 or response.status_code >= 300:
|
||||
if response.status_code in {408, 429, 500, 502, 503, 504}:
|
||||
raise ObservatoryWorkerHttpTransientError(
|
||||
f"Worker endpoint temporarily unavailable with HTTP {response.status_code}"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError(
|
||||
f"Worker endpoint rejected the request with HTTP {response.status_code}"
|
||||
)
|
||||
@@ -898,17 +904,13 @@ def _source_members(
|
||||
or document.get("claim_generation") != job.claim_generation
|
||||
or document.get("authority") != OBSERVATION_ONLY_AUTHORITY
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization belongs to another sealed job"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization belongs to another sealed job")
|
||||
source = _object(document.get("source"), "source materialization identity")
|
||||
if (
|
||||
set(source)
|
||||
!= {"session_id", "bundle_sha256", "capability_manifest_sha256"}
|
||||
set(source) != {"session_id", "bundle_sha256", "capability_manifest_sha256"}
|
||||
or source.get("session_id") != job.source_session_id
|
||||
or source.get("bundle_sha256") != job.source_bundle_sha256
|
||||
or source.get("capability_manifest_sha256")
|
||||
!= job.source_capability_manifest_sha256
|
||||
or source.get("capability_manifest_sha256") != job.source_capability_manifest_sha256
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("source materialization identity changed")
|
||||
values = document.get("members")
|
||||
@@ -919,8 +921,7 @@ def _source_members(
|
||||
len({member.member_id for member in members}) != len(members)
|
||||
or tuple(member.member_id for member in members)
|
||||
!= tuple(sorted(member.member_id for member in members))
|
||||
or sum(member.byte_length for member in members)
|
||||
> 2 * 1024 * 1024 * 1024 * 1024
|
||||
or sum(member.byte_length for member in members) > 2 * 1024 * 1024 * 1024 * 1024
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("source materialization bounds changed")
|
||||
for member in members:
|
||||
@@ -941,29 +942,18 @@ def _source_members(
|
||||
)
|
||||
).hexdigest()
|
||||
if member.member_id != expected_member_id:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization member identity changed"
|
||||
)
|
||||
bundle_members = tuple(
|
||||
member for member in members if member.kind == "source-bundle"
|
||||
)
|
||||
capability_members = tuple(
|
||||
member for member in members if member.kind == "source-capability"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization member identity changed")
|
||||
bundle_members = tuple(member for member in members if member.kind == "source-bundle")
|
||||
capability_members = tuple(member for member in members if member.kind == "source-capability")
|
||||
if (
|
||||
len(bundle_members) != 1
|
||||
or bundle_members[0].sha256 != job.source_bundle_sha256
|
||||
or len(capability_members) != 1
|
||||
or capability_members[0].sha256
|
||||
!= job.source_capability_manifest_sha256
|
||||
or capability_members[0].sha256 != job.source_capability_manifest_sha256
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("source materialization documents are incomplete")
|
||||
if sum(
|
||||
member.kind == "spatial-replay" and member.primary for member in members
|
||||
) != 1:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization primary replay is invalid"
|
||||
)
|
||||
if sum(member.kind == "spatial-replay" and member.primary for member in members) != 1:
|
||||
raise ObservatoryWorkerHttpError("source materialization primary replay is invalid")
|
||||
metadata_members = tuple(
|
||||
member for member in members if member.kind == "spatial-replay-metadata"
|
||||
)
|
||||
@@ -973,15 +963,9 @@ def _source_members(
|
||||
or member.primary
|
||||
for member in metadata_members
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization replay metadata is invalid"
|
||||
)
|
||||
camera_inits = tuple(
|
||||
member for member in members if member.kind == "camera-init"
|
||||
)
|
||||
camera_segments = tuple(
|
||||
member for member in members if member.kind == "camera-segment"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization replay metadata is invalid")
|
||||
camera_inits = tuple(member for member in members if member.kind == "camera-init")
|
||||
camera_segments = tuple(member for member in members if member.kind == "camera-segment")
|
||||
if (
|
||||
len(camera_inits) != 1
|
||||
or not camera_segments
|
||||
@@ -991,9 +975,7 @@ def _source_members(
|
||||
for member in camera_segments
|
||||
)
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization camera epoch is incomplete"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
|
||||
return members
|
||||
|
||||
|
||||
@@ -1053,9 +1035,7 @@ def _source_member(value: object) -> _SourceMember:
|
||||
or camera_epoch is not None
|
||||
or camera_sequence is not None
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source document member metadata is invalid"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source document member metadata is invalid")
|
||||
elif kind in {"spatial-replay", "spatial-replay-metadata"}:
|
||||
if (
|
||||
artifact_id is None
|
||||
@@ -1063,9 +1043,7 @@ def _source_member(value: object) -> _SourceMember:
|
||||
or camera_sequence is not None
|
||||
or (kind == "spatial-replay-metadata" and row["primary"])
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"spatial source member metadata is invalid"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("spatial source member metadata is invalid")
|
||||
elif kind == "camera-init":
|
||||
if (
|
||||
artifact_id is None
|
||||
@@ -1073,18 +1051,9 @@ def _source_member(value: object) -> _SourceMember:
|
||||
or camera_epoch is None
|
||||
or camera_sequence is not None
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera init member metadata is invalid"
|
||||
)
|
||||
elif (
|
||||
artifact_id is None
|
||||
or row["primary"]
|
||||
or camera_epoch is None
|
||||
or camera_sequence is None
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera segment member metadata is invalid"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("camera init member metadata is invalid")
|
||||
elif artifact_id is None or row["primary"] or camera_epoch is None or camera_sequence is None:
|
||||
raise ObservatoryWorkerHttpError("camera segment member metadata is invalid")
|
||||
return _SourceMember(
|
||||
member_id=member_id,
|
||||
kind=cast(
|
||||
@@ -1114,27 +1083,18 @@ def _ordered_camera_epoch_members(
|
||||
inits = tuple(member for member in members if member.kind == "camera-init")
|
||||
segments = tuple(member for member in members if member.kind == "camera-segment")
|
||||
if len(inits) != 1 or not segments:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization camera epoch is incomplete"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
|
||||
init = inits[0]
|
||||
if init.artifact_id is None or init.camera_epoch is None:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization camera identity is incomplete"
|
||||
)
|
||||
ordered_segments = tuple(
|
||||
sorted(segments, key=lambda member: member.camera_sequence or 0)
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization camera identity is incomplete")
|
||||
ordered_segments = tuple(sorted(segments, key=lambda member: member.camera_sequence or 0))
|
||||
if tuple(member.camera_sequence for member in ordered_segments) != tuple(
|
||||
range(1, len(ordered_segments) + 1)
|
||||
) or any(
|
||||
member.artifact_id != init.artifact_id
|
||||
or member.camera_epoch != init.camera_epoch
|
||||
member.artifact_id != init.artifact_id or member.camera_epoch != init.camera_epoch
|
||||
for member in ordered_segments
|
||||
):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization camera member order changed"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization camera member order changed")
|
||||
return (init, *ordered_segments)
|
||||
|
||||
|
||||
@@ -1147,13 +1107,9 @@ def _camera_archive_relative_name(member: _SourceMember) -> str:
|
||||
|
||||
|
||||
def _camera_epoch_archive_byte_length(members: tuple[_SourceMember, ...]) -> int:
|
||||
content_bytes = sum(
|
||||
512 + ((member.byte_length + 511) // 512) * 512 for member in members
|
||||
)
|
||||
content_bytes = sum(512 + ((member.byte_length + 511) // 512) * 512 for member in members)
|
||||
logical_bytes = content_bytes + 1024
|
||||
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (
|
||||
tarfile.RECORDSIZE
|
||||
)
|
||||
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (tarfile.RECORDSIZE)
|
||||
|
||||
|
||||
def _extract_camera_epoch_archive(
|
||||
@@ -1166,9 +1122,7 @@ def _extract_camera_epoch_archive(
|
||||
try:
|
||||
archive_metadata = archive_path.lstat()
|
||||
except OSError as exc:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive is unavailable"
|
||||
) from exc
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(archive_metadata.st_mode)
|
||||
or not stat.S_ISREG(archive_metadata.st_mode)
|
||||
@@ -1178,9 +1132,7 @@ def _extract_camera_epoch_archive(
|
||||
"camera epoch archive size differs from its member inventory"
|
||||
)
|
||||
transfer_root = _secure_directory(archive_path.parent)
|
||||
staging = Path(
|
||||
tempfile.mkdtemp(prefix=".camera-epoch-extract-", dir=transfer_root)
|
||||
)
|
||||
staging = Path(tempfile.mkdtemp(prefix=".camera-epoch-extract-", dir=transfer_root))
|
||||
staged: list[Path] = []
|
||||
archive_descriptor = -1
|
||||
try:
|
||||
@@ -1243,9 +1195,7 @@ def _extract_camera_epoch_archive(
|
||||
except ObservatoryWorkerHttpError:
|
||||
raise
|
||||
except (OSError, tarfile.TarError) as exc:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive could not be extracted"
|
||||
) from exc
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive could not be extracted") from exc
|
||||
finally:
|
||||
if archive_descriptor >= 0:
|
||||
os.close(archive_descriptor)
|
||||
@@ -1261,10 +1211,7 @@ def _copy_camera_archive_member(
|
||||
) -> None:
|
||||
descriptor = os.open(
|
||||
destination,
|
||||
os.O_WRONLY
|
||||
| os.O_CREAT
|
||||
| os.O_EXCL
|
||||
| getattr(os, "O_NOFOLLOW", 0),
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
|
||||
0o600,
|
||||
)
|
||||
digest = hashlib.sha256()
|
||||
@@ -1284,9 +1231,7 @@ def _copy_camera_archive_member(
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
if byte_length != expected_byte_length or digest.hexdigest() != expected_sha256:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"camera epoch archive member content changed"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("camera epoch archive member content changed")
|
||||
verified = True
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
@@ -1300,13 +1245,9 @@ def _prepare_source_destination_layout(
|
||||
root: Path,
|
||||
members: tuple[_SourceMember, ...],
|
||||
) -> _SourceDestinationLayout:
|
||||
camera_inits = tuple(
|
||||
member for member in members if member.kind == "camera-init"
|
||||
)
|
||||
camera_inits = tuple(member for member in members if member.kind == "camera-init")
|
||||
if len(camera_inits) != 1 or camera_inits[0].camera_epoch is None:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization camera epoch is incomplete"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
|
||||
camera_epoch = camera_inits[0].camera_epoch
|
||||
camera_root = _secure_source_subdirectory(root, root / "camera")
|
||||
camera_epoch_root = _secure_source_subdirectory(
|
||||
@@ -1318,10 +1259,7 @@ def _prepare_source_destination_layout(
|
||||
camera_epoch_root / "segments",
|
||||
)
|
||||
spatial = None
|
||||
if any(
|
||||
member.kind == "spatial-replay" and not member.primary
|
||||
for member in members
|
||||
):
|
||||
if any(member.kind == "spatial-replay" and not member.primary for member in members):
|
||||
spatial = _secure_source_subdirectory(root, root / "spatial")
|
||||
return _SourceDestinationLayout(
|
||||
root=root,
|
||||
@@ -1335,9 +1273,7 @@ def _prepare_source_destination_layout(
|
||||
def _secure_source_subdirectory(root: Path, candidate: Path) -> Path:
|
||||
resolved = _secure_directory(candidate)
|
||||
if not resolved.is_relative_to(root):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"source materialization destination escapes its root"
|
||||
)
|
||||
raise ObservatoryWorkerHttpError("source materialization destination escapes its root")
|
||||
return resolved
|
||||
|
||||
|
||||
@@ -1409,8 +1345,7 @@ def _upload_members(
|
||||
or len({member.role for member in members}) != len(members)
|
||||
or tuple(member.member_id for member in members)
|
||||
!= tuple(sorted(member.member_id for member in members))
|
||||
or sum(member.byte_length for member in members)
|
||||
> 256 * 1024 * 1024 * 1024
|
||||
or sum(member.byte_length for member in members) > 256 * 1024 * 1024 * 1024
|
||||
or document.get("complete") != all(member.uploaded for member in members)
|
||||
):
|
||||
raise ObservatoryWorkerHttpError("result upload member inventory is invalid")
|
||||
@@ -1489,9 +1424,7 @@ def _write_local_exact(path: Path, payload: bytes) -> None:
|
||||
os.chmod(path, 0o400, follow_symlinks=False)
|
||||
except FileExistsError:
|
||||
if _read_local_file(path, max(1, len(payload))) != payload:
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"Worker local manifest identity collided"
|
||||
) from None
|
||||
raise ObservatoryWorkerHttpError("Worker local manifest identity collided") from None
|
||||
finally:
|
||||
with suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
@@ -1512,9 +1445,7 @@ def _publish_local_file(
|
||||
os.chmod(destination, 0o400, follow_symlinks=False)
|
||||
except FileExistsError:
|
||||
if not _matches_file(destination, sha256, byte_length):
|
||||
raise ObservatoryWorkerHttpError(
|
||||
"Worker local member publication collided"
|
||||
) from None
|
||||
raise ObservatoryWorkerHttpError("Worker local member publication collided") from None
|
||||
|
||||
|
||||
def _matches_file(path: Path, sha256: str, byte_length: int) -> bool:
|
||||
@@ -1530,11 +1461,7 @@ def _read_local_file(path: Path, maximum_bytes: int) -> bytes:
|
||||
try:
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_size < 0
|
||||
or before.st_size > maximum_bytes
|
||||
):
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size < 0 or before.st_size > maximum_bytes:
|
||||
raise ObservatoryWorkerHttpError("Worker local file is outside bounds")
|
||||
payload = bytearray()
|
||||
while len(payload) < before.st_size:
|
||||
@@ -1562,11 +1489,7 @@ def _hash_local_regular_file(path: Path, maximum_bytes: int) -> tuple[str, int]:
|
||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
before = os.fstat(descriptor)
|
||||
if (
|
||||
not stat.S_ISREG(before.st_mode)
|
||||
or before.st_size < 0
|
||||
or before.st_size > maximum_bytes
|
||||
):
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size < 0 or before.st_size > maximum_bytes:
|
||||
raise ObservatoryWorkerHttpError("Worker local file is outside bounds")
|
||||
digest = hashlib.sha256()
|
||||
byte_length = 0
|
||||
|
||||
@@ -52,6 +52,7 @@ from k1link.observatory.worker_http_transport import (
|
||||
OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"
|
||||
OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"
|
||||
OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"
|
||||
OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CACHE_ROOT"
|
||||
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS"
|
||||
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = (
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS"
|
||||
@@ -140,11 +141,14 @@ class ObservatoryWorkerServiceConfiguration:
|
||||
idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS
|
||||
transport_backoff_seconds: float = DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS
|
||||
max_consecutive_transport_failures: int = DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES
|
||||
source_cache_root: Path | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_validate_worker_base_url(self.base_url)
|
||||
_absolute_path(self.bearer_token_file, "Worker bearer token file")
|
||||
_absolute_path(self.work_root, "Worker work root")
|
||||
if self.source_cache_root is not None:
|
||||
_absolute_path(self.source_cache_root, "Worker source cache root")
|
||||
if isinstance(self.idle_poll_seconds, bool) or not (
|
||||
0.05 <= self.idle_poll_seconds <= 300.0
|
||||
):
|
||||
@@ -175,6 +179,11 @@ class ObservatoryWorkerServiceConfiguration:
|
||||
),
|
||||
bearer_token_file=token_file,
|
||||
work_root=work_root,
|
||||
source_cache_root=(
|
||||
_required_path(values, OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV)
|
||||
if values.get(OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV)
|
||||
else None
|
||||
),
|
||||
idle_poll_seconds=_environment_float(
|
||||
values,
|
||||
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV,
|
||||
@@ -265,6 +274,7 @@ def compose_installed_observatory_worker_service(
|
||||
bearer_token=bearer_token,
|
||||
work_root=configuration.work_root,
|
||||
transport=http_transport,
|
||||
source_cache_root=configuration.source_cache_root,
|
||||
)
|
||||
finally:
|
||||
# The immutable string remains owned by the gateway headers for the
|
||||
@@ -302,6 +312,7 @@ def compose_installed_observatory_worker_service_from_builders(
|
||||
bearer_token=bearer_token,
|
||||
work_root=configuration.work_root,
|
||||
transport=http_transport,
|
||||
source_cache_root=configuration.source_cache_root,
|
||||
)
|
||||
executors = build_ready_executor_registry(
|
||||
definitions=definitions,
|
||||
@@ -343,6 +354,7 @@ def compose_installed_observatory_worker_service_from_packages(
|
||||
bearer_token=bearer_token,
|
||||
work_root=configuration.work_root,
|
||||
transport=http_transport,
|
||||
source_cache_root=configuration.source_cache_root,
|
||||
)
|
||||
executors = build_ready_executor_registry_from_packages(
|
||||
definitions=definitions,
|
||||
@@ -381,9 +393,7 @@ def build_ready_executor_registry_from_packages(
|
||||
_absolute_path(work_root, "Worker package build work root")
|
||||
ready = definitions.ready_recorded_definitions()
|
||||
ready_keys = {(item.setup_id, item.definition_sha256) for item in ready}
|
||||
package_keys = {
|
||||
(package.setup_id, package.definition_sha256) for package in packages.packages
|
||||
}
|
||||
package_keys = {(package.setup_id, package.definition_sha256) for package in packages.packages}
|
||||
if not package_keys.issubset(ready_keys):
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"installed LAB packages must bind only ready RunDefinitions"
|
||||
@@ -420,7 +430,7 @@ def build_ready_executor_registry_from_packages(
|
||||
if built.identity != package.executor_identity:
|
||||
raise ObservatoryWorkerServiceError(
|
||||
"generic package factory returned another executor identity"
|
||||
)
|
||||
)
|
||||
registrations.append(built)
|
||||
return ObservatoryWorkerExecutorRegistry(tuple(registrations))
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Worker-local reuse of immutable source bytes, never of claim authority.
|
||||
|
||||
Each job must still obtain and validate its own current source manifest. Only
|
||||
exact digest/length matches may populate its fixed, generation-scoped layout.
|
||||
Files use read-only hardlinks on the same filesystem. An explicitly shared cache
|
||||
on another filesystem uses bounded verified disk copies, never whole-file RAM.
|
||||
An absent or damaged cache object is a miss; it is never overwritten or deleted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import stat
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_CHUNK_BYTES = 1024 * 1024
|
||||
_FREE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
|
||||
|
||||
|
||||
class WorkerSourceCacheError(RuntimeError):
|
||||
"""A caller selected an unsafe cache root, identity or destination."""
|
||||
|
||||
|
||||
class WorkerSourceCache:
|
||||
def __init__(self, root: Path) -> None:
|
||||
candidate = root.expanduser().absolute()
|
||||
candidate.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if candidate.is_symlink() or not candidate.is_dir():
|
||||
raise WorkerSourceCacheError("Worker source cache root is unsafe")
|
||||
self.root = candidate.resolve(strict=True)
|
||||
|
||||
def restore(self, destination: Path, *, sha256: str, byte_length: int) -> bool:
|
||||
"""Verify cached bytes before linking them into an empty job-owned path."""
|
||||
cached = self._path(sha256, byte_length)
|
||||
if not cached.exists() or cached.is_symlink():
|
||||
return False
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise WorkerSourceCacheError("Worker source cache destination is not empty")
|
||||
return _publish_exact(cached, destination, sha256, byte_length)
|
||||
|
||||
def retain(self, source: Path, *, sha256: str, byte_length: int) -> bool:
|
||||
"""Retain a fully downloaded/extracted file without copying its contents.
|
||||
|
||||
Corrupt existing cache files are preserved for diagnosis, not used and
|
||||
not replaced. The current run can use its freshly downloaded exact file.
|
||||
"""
|
||||
cached = self._path(sha256, byte_length)
|
||||
if cached.exists() or cached.is_symlink():
|
||||
# Not consumed by this call. Restore will validate it when needed;
|
||||
# do not add another whole-file read to an already-ready job.
|
||||
return False
|
||||
return _publish_exact(source, cached, sha256, byte_length)
|
||||
|
||||
def _path(self, sha256: str, byte_length: int) -> Path:
|
||||
if (
|
||||
not isinstance(sha256, str)
|
||||
or _SHA256.fullmatch(sha256) is None
|
||||
or isinstance(byte_length, bool)
|
||||
or not isinstance(byte_length, int)
|
||||
or byte_length < 0
|
||||
):
|
||||
raise WorkerSourceCacheError("Worker source cache identity is invalid")
|
||||
if self.root.is_symlink() or not self.root.is_dir():
|
||||
raise WorkerSourceCacheError("Worker source cache root changed")
|
||||
return self.root / sha256
|
||||
|
||||
|
||||
def _publish_exact(source: Path, destination: Path, sha256: str, byte_length: int) -> bool:
|
||||
"""Hash actual staged bytes before publication; never replace a path."""
|
||||
temporary = destination.with_name(f".source-cache-{secrets.token_hex(16)}")
|
||||
try:
|
||||
try:
|
||||
os.link(source, temporary, follow_symlinks=False)
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EXDEV:
|
||||
return False
|
||||
# Different agent work mounts may share one CAS mount. Copy only
|
||||
# in bounded chunks and leave a cache miss when space is low.
|
||||
if shutil.disk_usage(destination.parent).free < byte_length + _FREE_RESERVE_BYTES:
|
||||
return False
|
||||
if not _copy_exact(source, temporary, sha256, byte_length):
|
||||
return False
|
||||
else:
|
||||
if not _matches(temporary, sha256, byte_length):
|
||||
return False
|
||||
os.chmod(temporary, 0o400, follow_symlinks=False)
|
||||
try:
|
||||
os.link(temporary, destination, follow_symlinks=False)
|
||||
except FileExistsError:
|
||||
return _matches(destination, sha256, byte_length)
|
||||
return True
|
||||
except OSError:
|
||||
# Cache publication is optional; a fully downloaded job remains valid.
|
||||
return False
|
||||
finally:
|
||||
with suppress(FileNotFoundError):
|
||||
temporary.unlink() # Only this call's temporary link/copy.
|
||||
|
||||
|
||||
def _matches(path: Path, sha256: str, byte_length: int) -> bool:
|
||||
descriptor = -1
|
||||
try:
|
||||
if not stat.S_ISREG(path.lstat().st_mode):
|
||||
return False
|
||||
descriptor = os.open(
|
||||
path,
|
||||
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
|
||||
)
|
||||
before = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size != byte_length:
|
||||
return False
|
||||
digest = hashlib.sha256()
|
||||
remaining = byte_length
|
||||
while remaining:
|
||||
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
|
||||
if not chunk:
|
||||
return False
|
||||
digest.update(chunk)
|
||||
remaining -= len(chunk)
|
||||
after = os.fstat(descriptor)
|
||||
named = path.lstat()
|
||||
return (
|
||||
stat.S_ISREG(named.st_mode)
|
||||
and _stamp(before) == _stamp(after) == _stamp(named)
|
||||
and digest.hexdigest() == sha256
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
finally:
|
||||
if descriptor >= 0:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _stamp(value: os.stat_result) -> tuple[int, int, int, int, int]:
|
||||
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns
|
||||
|
||||
|
||||
def _copy_exact(source: Path, temporary: Path, sha256: str, byte_length: int) -> bool:
|
||||
if not stat.S_ISREG(source.lstat().st_mode):
|
||||
return False
|
||||
descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
before = os.fstat(descriptor)
|
||||
if not stat.S_ISREG(before.st_mode) or before.st_size != byte_length:
|
||||
return False
|
||||
digest = hashlib.sha256()
|
||||
remaining = byte_length
|
||||
with temporary.open("xb") as target:
|
||||
while remaining:
|
||||
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
|
||||
if not chunk:
|
||||
return False
|
||||
digest.update(chunk)
|
||||
target.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
target.flush()
|
||||
os.fsync(target.fileno())
|
||||
valid = _stamp(before) == _stamp(os.fstat(descriptor)) and digest.hexdigest() == sha256
|
||||
if not valid:
|
||||
temporary.unlink()
|
||||
return valid
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
@@ -19,9 +19,7 @@ from .models import ObservationSessionCandidate
|
||||
|
||||
EQUIPMENT_REGISTRY_SCHEMA: Final = "missioncore.equipment-model-registry/v1"
|
||||
EQUIPMENT_MODEL_SCHEMA: Final = "missioncore.equipment-model/v1"
|
||||
CAPTURE_PROFILE_REGISTRY_SCHEMA: Final = (
|
||||
"missioncore.recorded-capture-profile-registry/v1"
|
||||
)
|
||||
CAPTURE_PROFILE_REGISTRY_SCHEMA: Final = "missioncore.recorded-capture-profile-registry/v1"
|
||||
CAPTURE_PROFILE_SCHEMA: Final = "missioncore.recorded-capture-profile/v1"
|
||||
CAPTURE_ATTESTATION_SCHEMA: Final = "missioncore.recorded-capture-attestation/v1"
|
||||
|
||||
@@ -119,9 +117,7 @@ class EquipmentCaptureRegistry:
|
||||
@classmethod
|
||||
def from_repository(cls, repository_root: Path) -> EquipmentCaptureRegistry | None:
|
||||
equipment_path = repository_root / "config" / "observatory-equipment-models.json"
|
||||
capture_path = (
|
||||
repository_root / "config" / "observatory-recorded-capture-profiles.json"
|
||||
)
|
||||
capture_path = repository_root / "config" / "observatory-recorded-capture-profiles.json"
|
||||
if not equipment_path.exists() and not capture_path.exists():
|
||||
return None
|
||||
if not equipment_path.is_file() or not capture_path.is_file():
|
||||
@@ -152,9 +148,7 @@ class EquipmentCaptureRegistry:
|
||||
if len(by_id) != len(equipment_models):
|
||||
raise EquipmentRegistryError("equipment model ids must be unique")
|
||||
capture_profiles = tuple(_capture_profile(row, by_id) for row in capture_rows)
|
||||
if len({item.capture_profile_id for item in capture_profiles}) != len(
|
||||
capture_profiles
|
||||
):
|
||||
if len({item.capture_profile_id for item in capture_profiles}) != len(capture_profiles):
|
||||
raise EquipmentRegistryError("capture profile ids must be unique")
|
||||
return cls(
|
||||
equipment_models=equipment_models,
|
||||
@@ -227,26 +221,28 @@ class EquipmentCaptureRegistry:
|
||||
row = profile.document
|
||||
camera = row["camera_media"]
|
||||
calibration = row["calibration"]
|
||||
modalities = row["modalities"]
|
||||
semantic_channels = row["semantic_channels"]
|
||||
assert isinstance(camera, dict)
|
||||
assert isinstance(calibration, dict)
|
||||
assert isinstance(modalities, list)
|
||||
assert isinstance(semantic_channels, list)
|
||||
required_modalities = requirements.get("required_modalities")
|
||||
if (
|
||||
row["plugin_id"] == requirements.get("plugin_id")
|
||||
and row["archive_id"] == requirements.get("archive_id")
|
||||
and row["modalities"] == requirements.get("required_modalities")
|
||||
and isinstance(required_modalities, list)
|
||||
and set(required_modalities).issubset(modalities)
|
||||
and camera["source_id"] == requirements.get("camera_source_id")
|
||||
and "camera.video.recorded" in semantic_channels
|
||||
and requirements.get("camera_semantic_channel_id")
|
||||
== "camera.video.recorded"
|
||||
and requirements.get("camera_semantic_channel_id") == "camera.video.recorded"
|
||||
and camera["media_type"] == requirements.get("recorded_media_type")
|
||||
and camera["initialization_sha256"]
|
||||
== requirements.get("recorded_media_init_sha256")
|
||||
and camera["width"] == requirements.get("camera_width")
|
||||
and camera["height"] == requirements.get("camera_height")
|
||||
and calibration["slot_id"] == requirements.get("calibration_slot")
|
||||
and calibration["sha256"]
|
||||
== requirements.get("calibration_identity_sha256")
|
||||
and calibration["sha256"] == requirements.get("calibration_identity_sha256")
|
||||
and requirements.get("exactly_one_media_epoch") is True
|
||||
and requirements.get("seekable") is True
|
||||
):
|
||||
@@ -336,8 +332,7 @@ def _capture_profile(
|
||||
or not channels
|
||||
or channels != sorted(channels)
|
||||
or any(
|
||||
not isinstance(item, str) or _IDENTIFIER.fullmatch(item) is None
|
||||
for item in channels
|
||||
not isinstance(item, str) or _IDENTIFIER.fullmatch(item) is None for item in channels
|
||||
)
|
||||
):
|
||||
raise EquipmentRegistryError("capture semantic channels are invalid")
|
||||
|
||||
@@ -2,17 +2,20 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from threading import Lock
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID
|
||||
from uuid import UUID, uuid5
|
||||
|
||||
import rerun as rr
|
||||
import rerun_bindings as bindings
|
||||
from rerun import blueprint as rrb
|
||||
|
||||
from k1link.viewer.recorded_blueprint_lifecycle import (
|
||||
BlueprintSessionReleased,
|
||||
RecordedBlueprintSessions,
|
||||
)
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
||||
|
||||
APPLICATION_ID = "nodedc_mission_core_recorded"
|
||||
@@ -48,8 +51,9 @@ class _RecordedBlueprintStream:
|
||||
"""Keep one bounded SDK blueprint source for each browser viewport.
|
||||
|
||||
Upstream 0.36.3 activates a clone, not this source store. Explicit refresh
|
||||
makes layer changes visible but cannot retain the clone's operator eye.
|
||||
It is opt-in for portable replay pending native camera-state support.
|
||||
makes layer changes visible. Ordinary layer updates deliberately omit eye
|
||||
components so the active clone retains the operator's camera. Only an
|
||||
explicit 3D/plan/follow transition may write a new spatial eye preset.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -57,8 +61,10 @@ class _RecordedBlueprintStream:
|
||||
application_id: str,
|
||||
recording_id: str,
|
||||
*,
|
||||
blueprint_session_id: str,
|
||||
view_reset_generation: Literal[0, 1],
|
||||
) -> None:
|
||||
self.blueprint_session_id = blueprint_session_id
|
||||
self.view_reset_generation = view_reset_generation
|
||||
self._lock = Lock()
|
||||
self._sequence = 0
|
||||
@@ -81,7 +87,7 @@ class _RecordedBlueprintStream:
|
||||
|
||||
def render(
|
||||
self,
|
||||
blueprint_factory: Callable[[bool, bool], rrb.Blueprint],
|
||||
blueprint_factory: Callable[[bool, bool, str], rrb.Blueprint],
|
||||
*,
|
||||
follow_trajectory: bool,
|
||||
plan_view: bool,
|
||||
@@ -94,10 +100,20 @@ class _RecordedBlueprintStream:
|
||||
update_eye_controls = self._eye_contract != eye_contract
|
||||
# Initial admission/reset uses native framing. Explicit presets
|
||||
# apply to mode transitions, after the viewer has a source cursor.
|
||||
blueprint = blueprint_factory(update_eye_controls, self._eye_contract is not None)
|
||||
# Keep one view identity for the lifetime of this browser owner.
|
||||
# Replacing the UUID on every layer toggle rebuilt both native
|
||||
# viewports, delayed a simple visibility change, and discarded the
|
||||
# operator's active interaction state. The explicit reset
|
||||
# generation already owns the one intentional identity change.
|
||||
view_instance_token = self.blueprint_session_id
|
||||
blueprint = blueprint_factory(
|
||||
update_eye_controls,
|
||||
self._eye_contract is not None,
|
||||
view_instance_token,
|
||||
)
|
||||
# Appending rows alone does not refresh upstream's active clone.
|
||||
# Keep legacy admission unchanged; portable replay opts into
|
||||
# working layer updates with an explicitly documented eye reset.
|
||||
# Reactivation makes layer changes visible; eye components remain
|
||||
# absent unless the explicit view-mode contract changed above.
|
||||
make_active = self._sequence == 0 or reactivate_updates
|
||||
self._blueprint_recording.set_time(
|
||||
"blueprint",
|
||||
@@ -127,40 +143,7 @@ class _RecordedBlueprintStream:
|
||||
self._transport_recording.disconnect()
|
||||
|
||||
|
||||
_MAX_RECORDED_BLUEPRINT_STREAMS = 32
|
||||
_recorded_blueprint_streams_lock = Lock()
|
||||
_recorded_blueprint_streams: OrderedDict[tuple[str, str, str], _RecordedBlueprintStream] = (
|
||||
OrderedDict()
|
||||
)
|
||||
|
||||
|
||||
def _stable_recorded_blueprint_stream(
|
||||
*,
|
||||
application_id: str,
|
||||
recording_id: str,
|
||||
blueprint_session_id: str,
|
||||
view_reset_generation: Literal[0, 1],
|
||||
) -> _RecordedBlueprintStream:
|
||||
key = (application_id, recording_id, blueprint_session_id)
|
||||
with _recorded_blueprint_streams_lock:
|
||||
stream = _recorded_blueprint_streams.get(key)
|
||||
if stream is not None and stream.view_reset_generation != view_reset_generation:
|
||||
stream.close()
|
||||
del _recorded_blueprint_streams[key]
|
||||
stream = None
|
||||
if stream is None:
|
||||
stream = _RecordedBlueprintStream(
|
||||
application_id,
|
||||
recording_id,
|
||||
view_reset_generation=view_reset_generation,
|
||||
)
|
||||
_recorded_blueprint_streams[key] = stream
|
||||
else:
|
||||
_recorded_blueprint_streams.move_to_end(key)
|
||||
while len(_recorded_blueprint_streams) > _MAX_RECORDED_BLUEPRINT_STREAMS:
|
||||
_, stale = _recorded_blueprint_streams.popitem(last=False)
|
||||
stale.close()
|
||||
return stream
|
||||
recorded_blueprint_sessions = RecordedBlueprintSessions[_RecordedBlueprintStream]()
|
||||
|
||||
|
||||
def recorded_blueprint(
|
||||
@@ -170,15 +153,21 @@ def recorded_blueprint(
|
||||
active_view: RecordedView = "spatial",
|
||||
view_reset_generation: Literal[0, 1] = 0,
|
||||
unified_perception: bool = False,
|
||||
unified_camera_share: float = 0.46,
|
||||
semantic_layer: Literal["city", "vegetation"] | None = None,
|
||||
plan_view: bool = False,
|
||||
show_detections_2d: bool = False,
|
||||
show_camera_image: bool = True,
|
||||
show_segmentation: bool = False,
|
||||
show_cuboids_3d: bool = False,
|
||||
show_costmap: bool = False,
|
||||
follow_trajectory: bool = False,
|
||||
update_eye_controls: bool = True,
|
||||
explicit_spatial_preset: bool = False,
|
||||
eye_position: tuple[float, float, float] | None = None,
|
||||
eye_look_target: tuple[float, float, float] | None = None,
|
||||
eye_up: tuple[float, float, float] | None = None,
|
||||
view_instance_token: str | None = None,
|
||||
) -> rrb.Blueprint:
|
||||
accumulation = max(0.0, settings.accumulation_seconds)
|
||||
time_ranges: list[rr.VisibleTimeRange] | None = None
|
||||
@@ -218,6 +207,14 @@ def recorded_blueprint(
|
||||
)
|
||||
spatial_eye_controls = (
|
||||
rrb.EyeControls3D.from_fields(
|
||||
kind=rrb.Eye3DKind.Orbital,
|
||||
position=eye_position,
|
||||
look_target=eye_look_target,
|
||||
eye_up=eye_up,
|
||||
tracking_entity="/world/sensor_pose" if follow_trajectory else "",
|
||||
)
|
||||
if eye_position is not None and eye_look_target is not None and eye_up is not None
|
||||
else rrb.EyeControls3D.from_fields(
|
||||
kind=rrb.Eye3DKind.Orbital,
|
||||
position=[0.0, 0.0, 30.0],
|
||||
look_target=[0.0, 0.0, 0.0],
|
||||
@@ -267,15 +264,21 @@ def recorded_blueprint(
|
||||
# alone does not preserve edits in upstream's activated blueprint clone.
|
||||
eye_controls=spatial_eye_controls,
|
||||
)
|
||||
spatial_view.id = (
|
||||
RECORDED_SPATIAL_RESET_VIEW_ID if view_reset_generation else RECORDED_SPATIAL_VIEW_ID
|
||||
|
||||
def instance_id(primary: UUID, reset: UUID) -> UUID:
|
||||
base = reset if view_reset_generation else primary
|
||||
return uuid5(base, view_instance_token) if view_instance_token is not None else base
|
||||
|
||||
spatial_view.id = instance_id(
|
||||
RECORDED_SPATIAL_VIEW_ID,
|
||||
RECORDED_SPATIAL_RESET_VIEW_ID,
|
||||
)
|
||||
camera_view = rrb.Spatial2DView(
|
||||
origin="/perception/camera",
|
||||
name="Оригинальное видео · слои AI",
|
||||
background=[7, 8, 10, 255],
|
||||
overrides={
|
||||
"/perception/camera/image": rrb.EntityBehavior(visible=True),
|
||||
"/perception/camera/image": rrb.EntityBehavior(visible=show_camera_image),
|
||||
"/perception/camera/detections": rrb.EntityBehavior(
|
||||
visible=show_detections_2d,
|
||||
),
|
||||
@@ -290,8 +293,9 @@ def recorded_blueprint(
|
||||
),
|
||||
},
|
||||
)
|
||||
camera_view.id = (
|
||||
RECORDED_CAMERA_RESET_VIEW_ID if view_reset_generation else RECORDED_CAMERA_VIEW_ID
|
||||
camera_view.id = instance_id(
|
||||
RECORDED_CAMERA_VIEW_ID,
|
||||
RECORDED_CAMERA_RESET_VIEW_ID,
|
||||
)
|
||||
perception_3d_view = rrb.Spatial3DView(
|
||||
# Cuboids are expressed in the same calibrated world frame as the
|
||||
@@ -325,10 +329,9 @@ def recorded_blueprint(
|
||||
},
|
||||
eye_controls=spatial_eye_controls,
|
||||
)
|
||||
perception_3d_view.id = (
|
||||
RECORDED_PERCEPTION_3D_RESET_VIEW_ID
|
||||
if view_reset_generation
|
||||
else RECORDED_PERCEPTION_3D_VIEW_ID
|
||||
perception_3d_view.id = instance_id(
|
||||
RECORDED_PERCEPTION_3D_VIEW_ID,
|
||||
RECORDED_PERCEPTION_3D_RESET_VIEW_ID,
|
||||
)
|
||||
metrics_view = rrb.TimeSeriesView(
|
||||
origin="/metrics/device",
|
||||
@@ -357,16 +360,16 @@ def recorded_blueprint(
|
||||
spatial_view.visualizer_overrides["/world/perception/boxes3d"] = [
|
||||
rrb.EntityBehavior(visible=show_cuboids_3d),
|
||||
]
|
||||
camera_share = min(0.9, max(0.1, unified_camera_share))
|
||||
root_container = rrb.Horizontal(
|
||||
camera_view,
|
||||
spatial_view,
|
||||
column_shares=[0.46, 0.54],
|
||||
column_shares=[camera_share, 1.0 - camera_share],
|
||||
name="Единая сцена восприятия",
|
||||
)
|
||||
root_container.id = (
|
||||
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID
|
||||
if view_reset_generation
|
||||
else RECORDED_UNIFIED_ROOT_CONTAINER_ID
|
||||
root_container.id = instance_id(
|
||||
RECORDED_UNIFIED_ROOT_CONTAINER_ID,
|
||||
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID,
|
||||
)
|
||||
else:
|
||||
# Keep operator video and 3D cuboids as direct root views. Rerun's nested
|
||||
@@ -384,7 +387,7 @@ def recorded_blueprint(
|
||||
# replace it from a later blueprint message. Give every operator mode (and
|
||||
# its explicit reset generation) a stable root identity so the requested
|
||||
# child is authoritative instead of inheriting a previously visited tab.
|
||||
root_container.id = {
|
||||
base_root_id = {
|
||||
("spatial", 0): RECORDED_ROOT_CONTAINER_ID,
|
||||
("spatial", 1): RECORDED_SPATIAL_RESET_ROOT_CONTAINER_ID,
|
||||
("perception", 0): RECORDED_PERCEPTION_ROOT_CONTAINER_ID,
|
||||
@@ -394,6 +397,11 @@ def recorded_blueprint(
|
||||
("metrics", 0): RECORDED_METRICS_ROOT_CONTAINER_ID,
|
||||
("metrics", 1): RECORDED_METRICS_RESET_ROOT_CONTAINER_ID,
|
||||
}[(active_view, view_reset_generation)]
|
||||
root_container.id = (
|
||||
uuid5(base_root_id, view_instance_token)
|
||||
if view_instance_token is not None
|
||||
else base_root_id
|
||||
)
|
||||
|
||||
if include_initial_playback_state:
|
||||
return rrb.Blueprint(
|
||||
@@ -424,19 +432,26 @@ def recorded_blueprint_rrd(
|
||||
active_view: RecordedView = "spatial",
|
||||
view_reset_generation: Literal[0, 1] = 0,
|
||||
unified_perception: bool = False,
|
||||
unified_camera_share: float = 0.46,
|
||||
semantic_layer: Literal["city", "vegetation"] | None = None,
|
||||
plan_view: bool = False,
|
||||
show_detections_2d: bool = False,
|
||||
show_camera_image: bool = True,
|
||||
show_segmentation: bool = False,
|
||||
show_cuboids_3d: bool = False,
|
||||
show_costmap: bool = False,
|
||||
follow_trajectory: bool = False,
|
||||
reactivate_updates: bool = False,
|
||||
eye_position: tuple[float, float, float] | None = None,
|
||||
eye_look_target: tuple[float, float, float] | None = None,
|
||||
eye_up: tuple[float, float, float] | None = None,
|
||||
) -> bytes:
|
||||
"""Serialize a bounded active blueprint update without recorded data."""
|
||||
|
||||
def build_blueprint(
|
||||
update_eye_controls: bool, use_spatial_preset: bool = False
|
||||
update_eye_controls: bool,
|
||||
use_spatial_preset: bool = False,
|
||||
view_instance_token: str | None = None,
|
||||
) -> rrb.Blueprint:
|
||||
return recorded_blueprint(
|
||||
settings,
|
||||
@@ -444,32 +459,43 @@ def recorded_blueprint_rrd(
|
||||
active_view=active_view,
|
||||
view_reset_generation=view_reset_generation,
|
||||
unified_perception=unified_perception,
|
||||
unified_camera_share=unified_camera_share,
|
||||
semantic_layer=semantic_layer,
|
||||
plan_view=plan_view,
|
||||
show_detections_2d=show_detections_2d,
|
||||
show_camera_image=show_camera_image,
|
||||
show_segmentation=show_segmentation,
|
||||
show_cuboids_3d=show_cuboids_3d,
|
||||
show_costmap=show_costmap,
|
||||
follow_trajectory=follow_trajectory,
|
||||
update_eye_controls=update_eye_controls,
|
||||
explicit_spatial_preset=reactivate_updates and use_spatial_preset,
|
||||
explicit_spatial_preset=use_spatial_preset and update_eye_controls,
|
||||
eye_position=eye_position,
|
||||
eye_look_target=eye_look_target,
|
||||
eye_up=eye_up,
|
||||
view_instance_token=view_instance_token,
|
||||
)
|
||||
|
||||
payload: bytes | None
|
||||
if blueprint_session_id is not None:
|
||||
try:
|
||||
payload = _stable_recorded_blueprint_stream(
|
||||
application_id=application_id,
|
||||
recording_id=recording_id,
|
||||
blueprint_session_id=blueprint_session_id,
|
||||
view_reset_generation=view_reset_generation,
|
||||
).render(
|
||||
build_blueprint,
|
||||
follow_trajectory=follow_trajectory,
|
||||
plan_view=plan_view,
|
||||
reactivate_updates=reactivate_updates,
|
||||
payload = recorded_blueprint_sessions.use(
|
||||
(application_id, recording_id, blueprint_session_id),
|
||||
view_reset_generation,
|
||||
lambda: _RecordedBlueprintStream(
|
||||
application_id,
|
||||
recording_id,
|
||||
blueprint_session_id=blueprint_session_id,
|
||||
view_reset_generation=view_reset_generation,
|
||||
),
|
||||
lambda stream: stream.render(
|
||||
build_blueprint,
|
||||
follow_trajectory=follow_trajectory,
|
||||
plan_view=plan_view,
|
||||
reactivate_updates=reactivate_updates,
|
||||
),
|
||||
)
|
||||
except RecordedBlueprintError:
|
||||
except (RecordedBlueprintError, BlueprintSessionReleased):
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RecordedBlueprintError("failed to serialize stable recorded blueprint") from exc
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Ephemeral viewport resources: renew while mounted, release at termination."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from threading import RLock
|
||||
from typing import Protocol
|
||||
|
||||
BlueprintKey = tuple[str, str, str]
|
||||
|
||||
|
||||
class Closable(Protocol):
|
||||
def close(self) -> None: ...
|
||||
|
||||
|
||||
class BlueprintSessionReleased(RuntimeError):
|
||||
"""A late update cannot resurrect an explicitly released viewport."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Entry[Resource: Closable]:
|
||||
resource: Resource
|
||||
generation: int
|
||||
last_used: float
|
||||
|
||||
|
||||
class RecordedBlueprintSessions[Resource: Closable]:
|
||||
"""No TTL applies while a render is running; idle owners renew separately.
|
||||
|
||||
The small registry lock also serializes release with render. Thus the
|
||||
release acknowledgement means native resources have actually been closed,
|
||||
and a previously queued update is fenced by a lightweight tombstone.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ttl_seconds: float = 300.0,
|
||||
max_entries: int = 32,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self._ttl = ttl_seconds
|
||||
self._max_entries = max_entries
|
||||
self._clock = clock
|
||||
self._lock = RLock()
|
||||
self._entries: OrderedDict[BlueprintKey, _Entry[Resource]] = OrderedDict()
|
||||
self._released: OrderedDict[BlueprintKey, float] = OrderedDict()
|
||||
|
||||
def _expire(self, now: float) -> None:
|
||||
for key, entry in list(self._entries.items()):
|
||||
if now - entry.last_used >= self._ttl:
|
||||
del self._entries[key]
|
||||
entry.resource.close()
|
||||
for key, expires in list(self._released.items()):
|
||||
if expires <= now:
|
||||
del self._released[key]
|
||||
|
||||
def use[Result](
|
||||
self,
|
||||
key: BlueprintKey,
|
||||
generation: int,
|
||||
create: Callable[[], Resource],
|
||||
render: Callable[[Resource], Result],
|
||||
) -> Result:
|
||||
with self._lock:
|
||||
now = self._clock()
|
||||
self._expire(now)
|
||||
if key in self._released:
|
||||
raise BlueprintSessionReleased("recorded viewport has been released")
|
||||
entry = self._entries.get(key)
|
||||
if entry is not None and entry.generation != generation:
|
||||
del self._entries[key]
|
||||
entry.resource.close()
|
||||
entry = None
|
||||
if entry is None:
|
||||
entry = _Entry(create(), generation, now)
|
||||
self._entries[key] = entry
|
||||
self._entries.move_to_end(key)
|
||||
while len(self._entries) > self._max_entries:
|
||||
_, stale = self._entries.popitem(last=False)
|
||||
stale.resource.close()
|
||||
try:
|
||||
return render(entry.resource)
|
||||
except BaseException:
|
||||
# A failed operation must not retain its partial native store.
|
||||
del self._entries[key]
|
||||
entry.resource.close()
|
||||
raise
|
||||
finally:
|
||||
entry.last_used = self._clock()
|
||||
|
||||
def renew(self, key: BlueprintKey) -> bool:
|
||||
with self._lock:
|
||||
now = self._clock()
|
||||
self._expire(now)
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return False
|
||||
entry.last_used = now
|
||||
self._entries.move_to_end(key)
|
||||
return True
|
||||
|
||||
def release(self, key: BlueprintKey) -> None:
|
||||
with self._lock:
|
||||
now = self._clock()
|
||||
self._expire(now)
|
||||
entry = self._entries.pop(key, None)
|
||||
self._released[key] = now + self._ttl
|
||||
self._released.move_to_end(key)
|
||||
# Tombstones contain identities only, never SDK resources or data.
|
||||
while len(self._released) > 4096:
|
||||
self._released.popitem(last=False)
|
||||
if entry is not None:
|
||||
entry.resource.close()
|
||||
|
||||
def expire(self) -> None:
|
||||
with self._lock:
|
||||
self._expire(self._clock())
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
entries = list(self._entries.values())
|
||||
self._entries.clear()
|
||||
self._released.clear()
|
||||
for entry in entries:
|
||||
entry.resource.close()
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Scene-derived orbital camera bounds for recorded Rerun views."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pyarrow.compute as pc
|
||||
import rerun_bindings as bindings
|
||||
|
||||
SESSION_TIMELINE = "session_time"
|
||||
MIN_ORBIT_DISTANCE = 0.02
|
||||
MAX_ORBITAL_ZOOM_OUT_FACTOR = 5.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SpatialBoundsSeries:
|
||||
times_ns: np.ndarray
|
||||
lower: np.ndarray
|
||||
upper: np.ndarray
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def _recorded_spatial_bounds_index(
|
||||
path_text: str,
|
||||
byte_length: int,
|
||||
modified_ns: int,
|
||||
) -> dict[str, _SpatialBoundsSeries]:
|
||||
"""Build a small temporal bounds index for one immutable RRD generation.
|
||||
|
||||
Layer and follow buttons only change a blueprint. Re-decoding the complete
|
||||
source RRD for every click made those controls wait on archive I/O. The
|
||||
cache key includes the file generation fingerprint; cached values contain
|
||||
only timestamps and six floats per sample, never point-cloud payloads.
|
||||
"""
|
||||
|
||||
del byte_length, modified_ns # Generation identity is carried by the cache key.
|
||||
reader = bindings.RrdReaderInternal(path_text)
|
||||
recording = next(
|
||||
(entry for entry in reader.store_entries() if entry.kind == "recording"),
|
||||
None,
|
||||
)
|
||||
if recording is None:
|
||||
raise ValueError("recorded camera source has no recording store")
|
||||
|
||||
rows: dict[str, list[tuple[int, np.ndarray, np.ndarray]]] = {
|
||||
"/world/points": [],
|
||||
"/world/trajectory": [],
|
||||
}
|
||||
for chunk in reader.stream(recording):
|
||||
entity_path = str(chunk.entity_path)
|
||||
if entity_path not in rows:
|
||||
continue
|
||||
batch = chunk.to_record_batch()
|
||||
names = batch.column_names
|
||||
if SESSION_TIMELINE not in names:
|
||||
continue
|
||||
component = (
|
||||
"Points3D:positions" if entity_path == "/world/points" else "LineStrips3D:strips"
|
||||
)
|
||||
if component not in names:
|
||||
continue
|
||||
times = np.asarray(
|
||||
batch.column(names.index(SESSION_TIMELINE)).cast(pa.int64()),
|
||||
dtype=np.int64,
|
||||
)
|
||||
spatial = batch.column(names.index(component))
|
||||
for index, timestamp in enumerate(times):
|
||||
values = _spatial_values(
|
||||
spatial.slice(index, 1),
|
||||
nested=entity_path == "/world/trajectory",
|
||||
)
|
||||
if not values.size:
|
||||
continue
|
||||
finite = values[np.isfinite(values).all(axis=1)]
|
||||
if not finite.size:
|
||||
continue
|
||||
rows[entity_path].append(
|
||||
(
|
||||
int(timestamp),
|
||||
np.min(finite, axis=0),
|
||||
np.max(finite, axis=0),
|
||||
)
|
||||
)
|
||||
|
||||
result: dict[str, _SpatialBoundsSeries] = {}
|
||||
for entity_path, samples in rows.items():
|
||||
if not samples:
|
||||
continue
|
||||
samples.sort(key=lambda sample: sample[0])
|
||||
result[entity_path] = _SpatialBoundsSeries(
|
||||
times_ns=np.asarray([sample[0] for sample in samples], dtype=np.int64),
|
||||
lower=np.asarray([sample[1] for sample in samples], dtype=np.float32),
|
||||
upper=np.asarray([sample[2] for sample in samples], dtype=np.float32),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def recorded_orbital_radius_limit(
|
||||
recording_path: Path,
|
||||
*,
|
||||
current_time_ns: int,
|
||||
accumulation_seconds: float,
|
||||
show_points: bool,
|
||||
show_trajectory: bool,
|
||||
) -> float | None:
|
||||
"""Match Rerun 0.36.3's current scene-diagonal zoom-out limit.
|
||||
|
||||
The LAB 3D view roots its native mapping data at ``/world``. Points and
|
||||
trajectory are the source entities whose visible-time query changes with
|
||||
the operator's accumulation setting. Derived layers are deliberately not
|
||||
folded into this source contract: they use the same calibrated world frame
|
||||
and do not own navigation.
|
||||
"""
|
||||
|
||||
if current_time_ns < 0 or not math.isfinite(accumulation_seconds) or accumulation_seconds < 0:
|
||||
raise ValueError("invalid recorded camera query")
|
||||
wanted = {
|
||||
*(("/world/points",) if show_points else ()),
|
||||
*(("/world/trajectory",) if show_trajectory else ()),
|
||||
}
|
||||
if not wanted:
|
||||
return None
|
||||
path = recording_path.expanduser().absolute()
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ValueError("recorded camera source is unavailable")
|
||||
|
||||
stat = path.stat()
|
||||
series_by_entity = _recorded_spatial_bounds_index(
|
||||
str(path),
|
||||
stat.st_size,
|
||||
stat.st_mtime_ns,
|
||||
)
|
||||
|
||||
lower_ns = current_time_ns - round(accumulation_seconds * 1_000_000_000)
|
||||
lower = np.array([np.inf, np.inf, np.inf], dtype=np.float32)
|
||||
upper = np.array([-np.inf, -np.inf, -np.inf], dtype=np.float32)
|
||||
found = False
|
||||
for entity_path in wanted:
|
||||
series = series_by_entity.get(entity_path)
|
||||
if series is None:
|
||||
continue
|
||||
if accumulation_seconds > 0:
|
||||
selected = (series.times_ns >= lower_ns) & (series.times_ns <= current_time_ns)
|
||||
else:
|
||||
eligible_end = int(np.searchsorted(series.times_ns, current_time_ns, side="right"))
|
||||
if eligible_end == 0:
|
||||
continue
|
||||
latest_time = series.times_ns[eligible_end - 1]
|
||||
selected = series.times_ns == latest_time
|
||||
if not selected.any():
|
||||
continue
|
||||
lower = np.minimum(lower, np.min(series.lower[selected], axis=0))
|
||||
upper = np.maximum(upper, np.max(series.upper[selected], axis=0))
|
||||
found = True
|
||||
if not found:
|
||||
return None
|
||||
|
||||
# macaw::BoundingBox and Rerun's eye controller operate in f32.
|
||||
diagonal = np.float32(np.linalg.norm((upper - lower).astype(np.float32)))
|
||||
if not np.isfinite(diagonal) or diagonal <= 0:
|
||||
return None
|
||||
return float(
|
||||
max(
|
||||
np.float32(MIN_ORBIT_DISTANCE),
|
||||
np.float32(diagonal * np.float32(MAX_ORBITAL_ZOOM_OUT_FACTOR)),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _spatial_values(column: pa.Array, *, nested: bool) -> np.ndarray:
|
||||
flattened = pc.list_flatten(column)
|
||||
if nested:
|
||||
flattened = pc.list_flatten(flattened)
|
||||
if not pa.types.is_fixed_size_list(flattened.type) or flattened.type.list_size != 3:
|
||||
raise ValueError("recorded spatial component is not a 3D vector")
|
||||
# Flatten through Arrow rather than reading ``FixedSizeListArray.values``:
|
||||
# the latter exposes the complete backing buffer and ignores a sliced
|
||||
# array's logical offset.
|
||||
values = pc.list_flatten(flattened).to_numpy(zero_copy_only=False)
|
||||
if values.size == 0:
|
||||
return np.empty((0, 3), dtype=np.float32)
|
||||
return np.asarray(values, dtype=np.float32).reshape((-1, 3))
|
||||
+108
-64
@@ -42,11 +42,19 @@ from k1link.observatory import (
|
||||
ObservatoryRunPreparationLedger,
|
||||
load_observatory_run_preparation_ledger,
|
||||
)
|
||||
from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
|
||||
from k1link.observatory.domain_ontology import (
|
||||
ObservatoryDomainOntology,
|
||||
ObservatoryOntologyError,
|
||||
)
|
||||
from k1link.observatory.lab_view_profiles import LabViewProfileError, LabViewProfileStore
|
||||
from k1link.observatory.m49_queue_binding import (
|
||||
M49QueueBindingConfig,
|
||||
M49QueueBindingError,
|
||||
M49RecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.modular_composition import CompositionError, ModuleRegistry
|
||||
from k1link.observatory.modular_composition_store import ModularCompositionStore
|
||||
from k1link.observatory.portable_publication_reconciler import (
|
||||
PortablePublicationReconciler,
|
||||
)
|
||||
@@ -196,6 +204,7 @@ from k1link.web.map_api import (
|
||||
build_map_router,
|
||||
)
|
||||
from k1link.web.map_view_api import build_map_view_router
|
||||
from k1link.web.modular_observatory_api import build_modular_observatory_router
|
||||
from k1link.web.observatory_api import build_observatory_router
|
||||
from k1link.web.observatory_worker_api import (
|
||||
ObservatoryWorkerAuthentication,
|
||||
@@ -288,6 +297,37 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
|
||||
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
|
||||
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
|
||||
session_store = SessionStore(REPOSITORY_ROOT)
|
||||
OBSERVATORY_AI_COMPOSITIONS: ModularCompositionStore | None
|
||||
OBSERVATORY_AI_COMPOSITION_RUNS: CompositionRunStore | None
|
||||
OBSERVATORY_DOMAIN_ONTOLOGY: ObservatoryDomainOntology | None
|
||||
OBSERVATORY_LAB_VIEW_PROFILES: LabViewProfileStore | None
|
||||
try:
|
||||
OBSERVATORY_DOMAIN_ONTOLOGY = ObservatoryDomainOntology.from_file(
|
||||
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
|
||||
)
|
||||
OBSERVATORY_AI_COMPOSITIONS = ModularCompositionStore(
|
||||
session_store.data_dir / "observatory-ai-compositions",
|
||||
ModuleRegistry.from_file(REPOSITORY_ROOT / "config" / "observatory-ai-modules.json"),
|
||||
)
|
||||
OBSERVATORY_AI_COMPOSITION_RUNS = CompositionRunStore(
|
||||
session_store.data_dir / "observatory-ai-composition-runs"
|
||||
)
|
||||
OBSERVATORY_LAB_VIEW_PROFILES = LabViewProfileStore(
|
||||
session_store.data_dir / "observatory-lab-view-profiles"
|
||||
)
|
||||
except (
|
||||
CompositionError,
|
||||
CompositionRunError,
|
||||
LabViewProfileError,
|
||||
ObservatoryOntologyError,
|
||||
OSError,
|
||||
ValueError,
|
||||
):
|
||||
# A modular catalog failure cannot disable recordings, existing LABs or Legacy.
|
||||
OBSERVATORY_AI_COMPOSITIONS = None
|
||||
OBSERVATORY_AI_COMPOSITION_RUNS = None
|
||||
OBSERVATORY_DOMAIN_ONTOLOGY = None
|
||||
OBSERVATORY_LAB_VIEW_PROFILES = None
|
||||
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
|
||||
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
|
||||
@@ -322,9 +362,7 @@ def _resolve_observatory_calculation_profile(
|
||||
summary: SessionSummary,
|
||||
) -> dict[str, object] | None:
|
||||
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
|
||||
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
|
||||
summary
|
||||
)
|
||||
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(summary)
|
||||
if legacy is not None:
|
||||
return legacy
|
||||
if (
|
||||
@@ -437,47 +475,30 @@ try:
|
||||
or "portable definition registry is unavailable"
|
||||
)
|
||||
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable calculation profile registry is unavailable"
|
||||
)
|
||||
raise PortableWorkerIntegrationError("portable calculation profile registry is unavailable")
|
||||
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"portable result validator registry is unavailable"
|
||||
)
|
||||
raise PortableWorkerIntegrationError("portable result validator registry is unavailable")
|
||||
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
|
||||
or "Observatory recorded-job queue is unavailable"
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR or "Observatory recorded-job queue is unavailable"
|
||||
)
|
||||
if session_artifact_gateway is None:
|
||||
raise PortableWorkerIntegrationError(
|
||||
"central artifact store is not configured"
|
||||
)
|
||||
raise PortableWorkerIntegrationError("central artifact store is not configured")
|
||||
if session_artifact_gateway.status().central_status != "ready":
|
||||
raise PortableWorkerIntegrationError(
|
||||
"central artifact store is unavailable"
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
|
||||
PortableWorkerStorageRoots.from_environment(
|
||||
artifact_store_root=session_artifact_gateway.store.root,
|
||||
)
|
||||
raise PortableWorkerIntegrationError("central artifact store is unavailable")
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = PortableWorkerStorageRoots.from_environment(
|
||||
artifact_store_root=session_artifact_gateway.store.root,
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
|
||||
build_portable_observatory_worker_integration(
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
artifact_store=session_artifact_gateway.store,
|
||||
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
|
||||
source_cas_root=(
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
|
||||
),
|
||||
result_staging_root=(
|
||||
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
|
||||
),
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = build_portable_observatory_worker_integration(
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
session_store=session_store,
|
||||
media_inspector=session_recorded_media_inspector,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
artifact_store=session_artifact_gateway.store,
|
||||
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
|
||||
source_cas_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root),
|
||||
result_staging_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root),
|
||||
)
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
|
||||
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||
@@ -487,10 +508,7 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
|
||||
OBSERVATORY_PUBLICATION_RECONCILER = (
|
||||
None
|
||||
if (
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE is None
|
||||
or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
|
||||
)
|
||||
if (OBSERVATORY_RECORDED_JOB_QUEUE is None or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None)
|
||||
else PortablePublicationReconciler(
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport,
|
||||
@@ -499,12 +517,10 @@ OBSERVATORY_PUBLICATION_RECONCILER = (
|
||||
)
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED
|
||||
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
)
|
||||
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = (
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED
|
||||
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
|
||||
)
|
||||
OBSERVATORY_WORKER_DISPATCH_READY = (
|
||||
OBSERVATORY_WORKER_CLAIM_LEASE_READY
|
||||
@@ -527,17 +543,13 @@ else:
|
||||
)
|
||||
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None:
|
||||
worker_api_errors.append(
|
||||
"authentication unavailable: "
|
||||
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||
f"authentication unavailable: {OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
|
||||
)
|
||||
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None:
|
||||
worker_api_errors.append(
|
||||
"integration unavailable: "
|
||||
f"{OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||
f"integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
|
||||
)
|
||||
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(
|
||||
worker_api_errors
|
||||
)
|
||||
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(worker_api_errors)
|
||||
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
|
||||
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
|
||||
@@ -554,7 +566,8 @@ try:
|
||||
and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
|
||||
):
|
||||
OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
|
||||
sessions=session_store, artifacts=session_artifact_gateway.store,
|
||||
sessions=session_store,
|
||||
artifacts=session_artifact_gateway.store,
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
|
||||
@@ -566,7 +579,8 @@ try:
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
published_result_available=(
|
||||
None if OBSERVATORY_PORTABLE_RESULT_CACHE is None
|
||||
None
|
||||
if OBSERVATORY_PORTABLE_RESULT_CACHE is None
|
||||
else OBSERVATORY_PORTABLE_RESULT_CACHE.available
|
||||
),
|
||||
)
|
||||
@@ -874,10 +888,19 @@ async def _portable_result_publication_reconciler() -> None:
|
||||
await asyncio.sleep(15.0)
|
||||
|
||||
|
||||
async def _recorded_blueprint_resource_reaper() -> None:
|
||||
from k1link.viewer.recorded import recorded_blueprint_sessions
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(30.0)
|
||||
await asyncio.to_thread(recorded_blueprint_sessions.expire)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler: asyncio.Task[None] | None = None
|
||||
publication_reconciler: asyncio.Task[None] | None = None
|
||||
blueprint_reaper: asyncio.Task[None] | None = None
|
||||
try:
|
||||
configure_scanner_diagnostics(session_store.data_dir / "logs")
|
||||
session_recording_preparation_manager.start()
|
||||
@@ -891,11 +914,17 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
# expensive on field captures. Start it immediately in the background
|
||||
# instead of holding the ASGI startup gate.
|
||||
reconciler = asyncio.create_task(_recording_preparation_reconciler())
|
||||
publication_reconciler = asyncio.create_task(
|
||||
_portable_result_publication_reconciler()
|
||||
)
|
||||
publication_reconciler = asyncio.create_task(_portable_result_publication_reconciler())
|
||||
blueprint_reaper = asyncio.create_task(_recorded_blueprint_resource_reaper())
|
||||
yield
|
||||
finally:
|
||||
from k1link.viewer.recorded import recorded_blueprint_sessions
|
||||
|
||||
if blueprint_reaper is not None:
|
||||
blueprint_reaper.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await blueprint_reaper
|
||||
await asyncio.to_thread(recorded_blueprint_sessions.close)
|
||||
await map_gateway_proxy.close()
|
||||
if reconciler is not None:
|
||||
reconciler.cancel()
|
||||
@@ -1066,7 +1095,9 @@ if session_artifact_gateway is not None and _ffmpeg is not None:
|
||||
media=session_recorded_media_inspector,
|
||||
recording_source=_canonical_lab_recording_source,
|
||||
ffmpeg_path=_ffmpeg,
|
||||
)
|
||||
),
|
||||
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1128,6 +1159,23 @@ app.include_router(
|
||||
),
|
||||
)
|
||||
)
|
||||
if (
|
||||
OBSERVATORY_AI_COMPOSITIONS is not None
|
||||
and OBSERVATORY_AI_COMPOSITION_RUNS is not None
|
||||
and OBSERVATORY_DOMAIN_ONTOLOGY is not None
|
||||
):
|
||||
app.include_router(
|
||||
build_modular_observatory_router(
|
||||
store=session_store,
|
||||
compositions=OBSERVATORY_AI_COMPOSITIONS,
|
||||
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
|
||||
ontology=OBSERVATORY_DOMAIN_ONTOLOGY,
|
||||
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
|
||||
binding=OBSERVATORY_PORTABLE_BINDING_SERVICE,
|
||||
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
view_profiles=OBSERVATORY_LAB_VIEW_PROFILES,
|
||||
)
|
||||
)
|
||||
if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
|
||||
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
|
||||
@@ -1136,12 +1184,8 @@ if OBSERVATORY_WORKER_DISPATCH_READY:
|
||||
build_observatory_worker_router(
|
||||
OBSERVATORY_RECORDED_JOB_QUEUE,
|
||||
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
|
||||
artifact_transport=(
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
|
||||
),
|
||||
result_publisher=(
|
||||
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
|
||||
),
|
||||
artifact_transport=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport),
|
||||
result_publisher=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from k1link.observatory.composition_runs import (
|
||||
CompositionRun,
|
||||
CompositionRunError,
|
||||
CompositionRunStore,
|
||||
)
|
||||
from k1link.observatory.domain_ontology import ObservatoryDomainOntology, ObservatoryOntologyError
|
||||
from k1link.observatory.lab_view_profiles import (
|
||||
PROFILE_SCHEMA as LAB_VIEW_PROFILE_SCHEMA,
|
||||
)
|
||||
from k1link.observatory.lab_view_profiles import (
|
||||
LabSceneProfile,
|
||||
LabViewProfile,
|
||||
LabViewProfileError,
|
||||
LabViewProfileStore,
|
||||
)
|
||||
from k1link.observatory.modular_composition import COMPOSITION_SCHEMA, CompositionError
|
||||
from k1link.observatory.modular_composition_store import ModularCompositionStore
|
||||
from k1link.observatory.portable_queue_binding import (
|
||||
PortableQueueBindingError,
|
||||
PortableRecordedQueueBindingService,
|
||||
)
|
||||
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueDuplicateError,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
from k1link.observatory.source_admission import PortableSourceNotPreparedError
|
||||
from k1link.sessions import SessionNotFoundError, SessionStore
|
||||
|
||||
_EXECUTABLE_SINGLE_MODULE_SETUPS = {
|
||||
"ddrnet": "ai-segmentation-ddrnet-v1",
|
||||
"eomt": "ai-segmentation-eomt-v1",
|
||||
"tgs": "m49-tgs-portable-v2",
|
||||
"rf-detr": "ai-detection-rf-detr-v1",
|
||||
"object-distance": "ai-range-object-distance-v1",
|
||||
}
|
||||
|
||||
|
||||
def _composition_error_detail(error: CompositionError) -> str:
|
||||
detail = str(error)
|
||||
if detail.startswith("unsupported value for "):
|
||||
return (
|
||||
"Параметры выбранного AI-модуля устарели. "
|
||||
"Закройте окно, откройте его снова и повторите расчёт."
|
||||
)
|
||||
if detail == "module version is not installed":
|
||||
return (
|
||||
"Версия выбранного AI-модуля обновилась. "
|
||||
"Закройте окно, откройте его снова и повторите расчёт."
|
||||
)
|
||||
if detail.startswith("select only one provider for "):
|
||||
return "В одном слое можно выбрать только один AI-модуль."
|
||||
if (
|
||||
detail.startswith("select a module providing ")
|
||||
or detail.startswith("ambiguous provider for ")
|
||||
or detail == "unresolved module dependencies"
|
||||
or detail == "cyclic module dependencies"
|
||||
):
|
||||
return "Для выбранной конфигурации не хватает обязательного связанного модуля."
|
||||
if detail == "select at least one AI module":
|
||||
return "Выберите хотя бы один AI-модуль."
|
||||
return "Конфигурацию AI-слоя не удалось проверить. Обновите окно и повторите выбор."
|
||||
|
||||
|
||||
class _Strict(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class AICompositionRequest(_Strict):
|
||||
schema_version: str
|
||||
source_session_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
selections: list[dict[str, Any]] = Field(min_length=1, max_length=6)
|
||||
idempotency_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=160,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$",
|
||||
)
|
||||
|
||||
|
||||
class AICompositionRunRenameRequest(_Strict):
|
||||
schema_version: str
|
||||
display_name: str = Field(min_length=1, max_length=160)
|
||||
|
||||
|
||||
class LabSceneProfileRequest(_Strict):
|
||||
point_size: float = Field(ge=0.1, allow_inf_nan=False)
|
||||
accumulation_seconds: float = Field(ge=0, allow_inf_nan=False)
|
||||
color_mode: str = Field(pattern=r"^(intensity|height|distance|rgb|class)$")
|
||||
palette: str = Field(pattern=r"^(turbo|viridis|plasma|grayscale)$")
|
||||
show_grid: bool
|
||||
show_labels: bool
|
||||
show_camera_frustums: bool
|
||||
|
||||
|
||||
class LabViewProfileRequest(_Strict):
|
||||
schema_version: str
|
||||
result_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$")
|
||||
scene_settings: LabSceneProfileRequest
|
||||
|
||||
|
||||
def build_modular_observatory_router(
|
||||
*,
|
||||
store: SessionStore,
|
||||
compositions: ModularCompositionStore,
|
||||
composition_runs: CompositionRunStore | None = None,
|
||||
ontology: ObservatoryDomainOntology | None = None,
|
||||
definitions: PortableRunDefinitionRegistry | None = None,
|
||||
binding: PortableRecordedQueueBindingService | None = None,
|
||||
queue: ObservatoryRecordedJobQueue | None = None,
|
||||
view_profiles: LabViewProfileStore | None = None,
|
||||
) -> APIRouter:
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/api/v1/observatory/ai-module-catalog")
|
||||
def catalog() -> dict[str, object]:
|
||||
return {
|
||||
**compositions.registry.catalog(),
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
@router.get("/api/v1/observatory/lab-view-profiles/{result_id}")
|
||||
def get_lab_view_profile(result_id: str) -> dict[str, object]:
|
||||
if view_profiles is None:
|
||||
raise HTTPException(503, "Профили отображения LAB недоступны.")
|
||||
try:
|
||||
profile = view_profiles.get(result_id)
|
||||
except LabViewProfileError as exc:
|
||||
raise HTTPException(422, "Некорректный профиль отображения LAB.") from exc
|
||||
if profile is None:
|
||||
raise HTTPException(404, "Профиль отображения LAB ещё не сохранён.")
|
||||
return profile.as_dict()
|
||||
|
||||
@router.put("/api/v1/observatory/lab-view-profiles/{result_id}")
|
||||
def put_lab_view_profile(result_id: str, request: LabViewProfileRequest) -> dict[str, object]:
|
||||
if view_profiles is None:
|
||||
raise HTTPException(503, "Профили отображения LAB недоступны.")
|
||||
if request.schema_version != LAB_VIEW_PROFILE_SCHEMA or request.result_id != result_id:
|
||||
raise HTTPException(422, "Профиль отображения относится к другой LAB.")
|
||||
try:
|
||||
settings = request.scene_settings
|
||||
profile = LabViewProfile(
|
||||
result_id=result_id,
|
||||
scene_settings=LabSceneProfile(**settings.model_dump()),
|
||||
updated_at_utc=datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
return view_profiles.save(profile).as_dict()
|
||||
except LabViewProfileError as exc:
|
||||
raise HTTPException(422, "Некорректные настройки отображения LAB.") from exc
|
||||
|
||||
@router.post("/api/v1/observatory/ai-compositions")
|
||||
def save(request: AICompositionRequest) -> dict[str, object]:
|
||||
if request.schema_version != COMPOSITION_SCHEMA:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail="Версия конфигурации AI-слоя не поддерживается.",
|
||||
)
|
||||
try:
|
||||
source = store.get_session(request.source_session_id).summary
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="Запись Обсерватории не найдена.") from exc
|
||||
if source.lab is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="AI-слой настраивается для исходной записи.",
|
||||
)
|
||||
selection = {"schema_version": request.schema_version, "selections": request.selections}
|
||||
try:
|
||||
composition, created = compositions.save(selection)
|
||||
except CompositionError as exc:
|
||||
raise HTTPException(status_code=409, detail=_composition_error_detail(exc)) from exc
|
||||
selected_modules = tuple(
|
||||
node.module.module_id
|
||||
for node in composition.nodes
|
||||
if node.module.group != "preparation"
|
||||
)
|
||||
selected = set(selected_modules)
|
||||
for previous in (
|
||||
()
|
||||
if composition_runs is None
|
||||
else composition_runs.list(source_session_id=source.session_id)
|
||||
):
|
||||
if previous.composition_sha256 != composition.sha256:
|
||||
continue
|
||||
try:
|
||||
previous_jobs = (
|
||||
tuple(queue.get(job_id) for job_id in previous.job_ids) if queue else ()
|
||||
)
|
||||
except (ObservatoryRecordedQueueError, ValueError):
|
||||
previous_jobs = ()
|
||||
if previous_jobs and all(
|
||||
job.state != "failed" and job.publication_state != "failed" for job in previous_jobs
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Эта конфигурация уже рассчитана или поставлена в очередь. "
|
||||
"Выберите другую конфигурацию."
|
||||
),
|
||||
)
|
||||
setup_ids: list[str] = []
|
||||
for module_id in ("ddrnet", "eomt", "tgs"):
|
||||
if module_id in selected:
|
||||
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS[module_id])
|
||||
if "object-distance" in selected:
|
||||
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS["object-distance"])
|
||||
elif "rf-detr" in selected:
|
||||
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS["rf-detr"])
|
||||
jobs = []
|
||||
reason = "Для этой композиции ещё не установлен исполняемый пакет Worker 006."
|
||||
if setup_ids and definitions is not None and binding is not None:
|
||||
try:
|
||||
checked = []
|
||||
existing_by_setup = {}
|
||||
for setup_id in setup_ids:
|
||||
definition = definitions.resolve_setup(setup_id)
|
||||
existing = None
|
||||
if queue is not None:
|
||||
candidates = queue.list_jobs(
|
||||
source_session_id=source.session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
limit=20,
|
||||
)
|
||||
existing = next(
|
||||
(job for job in candidates if job.state != "failed"),
|
||||
None,
|
||||
)
|
||||
if existing is not None:
|
||||
existing_by_setup[setup_id] = existing
|
||||
continue
|
||||
try:
|
||||
check = binding.check(
|
||||
source_session_id=source.session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
except PortableSourceNotPreparedError:
|
||||
check = binding.prepare_check(
|
||||
source_session_id=source.session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
)
|
||||
checked.append((setup_id, definition, check))
|
||||
for setup_id, definition, check in checked:
|
||||
key = hashlib.sha256(
|
||||
f"{request.idempotency_key}\0{setup_id}".encode()
|
||||
).hexdigest()
|
||||
job, _created = binding.submit(
|
||||
source_session_id=source.session_id,
|
||||
setup_id=setup_id,
|
||||
definition_sha256=definition.definition_sha256,
|
||||
expected_check_sha256=check.check_sha256,
|
||||
idempotency_key=f"ai-layer:{key}",
|
||||
)
|
||||
existing_by_setup[setup_id] = job
|
||||
jobs = [existing_by_setup[setup_id] for setup_id in setup_ids]
|
||||
if composition_runs is None and not checked:
|
||||
raise ObservatoryRecordedQueueDuplicateError("existing-composition")
|
||||
reason = (
|
||||
"Недостающие модули поставлены в очередь Worker 006; "
|
||||
"готовые результаты использованы повторно."
|
||||
if len(checked) < len(setup_ids)
|
||||
else "Композиция поставлена в очередь Worker 006."
|
||||
)
|
||||
except ObservatoryRecordedQueueDuplicateError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Эта конфигурация уже рассчитана или поставлена в очередь. "
|
||||
"Выберите другую конфигурацию."
|
||||
),
|
||||
) from exc
|
||||
except (PortableQueueBindingError, ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Композицию не удалось поставить в очередь Worker 006.",
|
||||
) from exc
|
||||
if not jobs and definitions is not None and binding is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Для этой конфигурации пока нет исполняемых модулей Worker 006.",
|
||||
)
|
||||
run_projection: dict[str, object] | None = None
|
||||
try:
|
||||
if composition_runs is None or ontology is None or not jobs:
|
||||
raise StopIteration
|
||||
run = composition_runs.save(
|
||||
source_session_id=source.session_id,
|
||||
composition=composition,
|
||||
setup_ids=tuple(setup_ids),
|
||||
job_ids=tuple(job.job_id for job in jobs),
|
||||
idempotency_key=request.idempotency_key,
|
||||
created_at_utc=datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
)
|
||||
presentation = ontology.project_composition(composition)
|
||||
run_projection = {**run.as_dict(), "presentation": presentation}
|
||||
except StopIteration:
|
||||
pass
|
||||
except (CompositionRunError, ObservatoryOntologyError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Связь композиции с результатами не удалось сохранить.",
|
||||
) from exc
|
||||
return {
|
||||
"schema_version": (
|
||||
"missioncore.observatory-ai-composition-receipt/v3"
|
||||
if run_projection is not None
|
||||
else "missioncore.observatory-ai-composition-receipt/v2"
|
||||
),
|
||||
"source_session_id": source.session_id,
|
||||
"composition": composition.as_dict(),
|
||||
"composition_sha256": composition.sha256,
|
||||
"created": created,
|
||||
**({"run": run_projection} if run_projection is not None else {}),
|
||||
"dispatch": {
|
||||
"ready": len(jobs) == len(setup_ids) and bool(jobs),
|
||||
"reason": reason,
|
||||
"setup_ids": setup_ids,
|
||||
"jobs": [job.as_dict() for job in jobs],
|
||||
},
|
||||
}
|
||||
|
||||
def project_run(run: CompositionRun) -> dict[str, object]:
|
||||
if ontology is None:
|
||||
raise CompositionRunError("composition ontology is unavailable")
|
||||
try:
|
||||
exact = composition_runs.get(run.run_id)
|
||||
jobs = [queue.get(job_id) for job_id in exact.job_ids] if queue else []
|
||||
except (CompositionRunError, ObservatoryRecordedQueueError, ValueError):
|
||||
raise
|
||||
# The immutable composition document is the authority for projection;
|
||||
# reconstruct the selected module projection from the run's sealed IDs.
|
||||
presentation = ontology.project_module_ids(exact.module_ids)
|
||||
published = bool(jobs) and all(
|
||||
job.state == "succeeded" and job.publication_state == "published" and job.result_id
|
||||
for job in jobs
|
||||
)
|
||||
failed = any(job.state == "failed" or job.publication_state == "failed" for job in jobs)
|
||||
return {
|
||||
**exact.as_dict(),
|
||||
"state": "ready" if published else "failed" if failed else "running",
|
||||
"configuration_label": (
|
||||
f"{store.get_session(exact.source_session_id).summary.display_name} · "
|
||||
f"{presentation['configuration_label']}"
|
||||
),
|
||||
"display_name": composition_runs.display_name(exact.run_id),
|
||||
"presentation": presentation,
|
||||
"jobs": [job.as_dict() for job in jobs],
|
||||
"result_ids": [job.result_id for job in jobs if job.result_id is not None],
|
||||
}
|
||||
|
||||
@router.get("/api/v1/observatory/ai-composition-runs")
|
||||
def composition_run_list(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
try:
|
||||
items = (
|
||||
[]
|
||||
if composition_runs is None
|
||||
else [
|
||||
project_run(run)
|
||||
for run in composition_runs.list(
|
||||
source_session_id=source_session_id,
|
||||
include_hidden=False,
|
||||
)
|
||||
]
|
||||
)
|
||||
except (
|
||||
CompositionRunError,
|
||||
ObservatoryRecordedQueueError,
|
||||
ObservatoryOntologyError,
|
||||
ValueError,
|
||||
) as exc:
|
||||
raise HTTPException(503, "Композиции AI-слоя недоступны.") from exc
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-ai-composition-run-list/v1",
|
||||
"items": items,
|
||||
}
|
||||
|
||||
@router.patch("/api/v1/observatory/ai-composition-runs/{run_id}")
|
||||
def rename_composition_run_projection(
|
||||
run_id: str,
|
||||
request: AICompositionRunRenameRequest,
|
||||
) -> dict[str, object]:
|
||||
if request.schema_version != "missioncore.observatory-ai-composition-run-rename/v1":
|
||||
raise HTTPException(422, "Версия переименования результата не поддерживается.")
|
||||
if composition_runs is None:
|
||||
raise HTTPException(503, "Композиции AI-слоя недоступны.")
|
||||
try:
|
||||
display_name = composition_runs.rename_projection(run_id, request.display_name)
|
||||
except CompositionRunError as exc:
|
||||
raise HTTPException(404, "Результат AI inference не найден.") from exc
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-ai-composition-run-projection/v1",
|
||||
"run_id": run_id,
|
||||
"display_name": display_name,
|
||||
}
|
||||
|
||||
@router.delete(
|
||||
"/api/v1/observatory/ai-composition-runs/{run_id}",
|
||||
status_code=204,
|
||||
)
|
||||
def delete_composition_run_projection(run_id: str) -> Response:
|
||||
if composition_runs is None:
|
||||
raise HTTPException(503, "Композиции AI-слоя недоступны.")
|
||||
try:
|
||||
composition_runs.delete_projection(run_id)
|
||||
except CompositionRunError as exc:
|
||||
raise HTTPException(404, "Результат AI inference не найден.") from exc
|
||||
return Response(status_code=204)
|
||||
|
||||
@router.get("/api/v1/observatory/ai-runs")
|
||||
def runs(
|
||||
source_session_id: str = Query(
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
|
||||
),
|
||||
) -> dict[str, object]:
|
||||
if queue is None:
|
||||
raise HTTPException(status_code=503, detail="Очередь AI-слоёв недоступна.")
|
||||
try:
|
||||
jobs = [
|
||||
job
|
||||
for setup_id in _EXECUTABLE_SINGLE_MODULE_SETUPS.values()
|
||||
for job in queue.list_jobs(
|
||||
source_session_id=source_session_id,
|
||||
setup_id=setup_id,
|
||||
limit=20,
|
||||
)
|
||||
]
|
||||
except (ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(status_code=503, detail="Очередь AI-слоёв недоступна.") from exc
|
||||
jobs.sort(key=lambda job: job.created_at_utc, reverse=True)
|
||||
return {
|
||||
"schema_version": "missioncore.observatory-recorded-job-list/v1",
|
||||
"items": [job.as_dict() for job in jobs[:20]],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"production_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
return router
|
||||
@@ -9,15 +9,33 @@ from fastapi import APIRouter, HTTPException, Path, Query, Response
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from k1link.laboratory.canonical_rerun_overlay import CanonicalLabOverlayError
|
||||
from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
|
||||
from k1link.observatory.portable_replay import PortableReplayService
|
||||
from k1link.observatory.portable_result_view import PortableResultViewError
|
||||
from k1link.observatory.recorded_jobs import (
|
||||
ObservatoryRecordedJobQueue,
|
||||
ObservatoryRecordedQueueError,
|
||||
)
|
||||
|
||||
ResultId = Annotated[str, Path(pattern=r"^m49-tgs-portable-review-[a-f0-9]{64}$")]
|
||||
ResultId = Annotated[
|
||||
str,
|
||||
Path(
|
||||
pattern=(
|
||||
r"^(?:m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
|
||||
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance))-[a-f0-9]{64}$"
|
||||
)
|
||||
),
|
||||
]
|
||||
BaseSha = Annotated[str, Path(pattern=r"^[a-f0-9]{64}$")]
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_portable_replay_router(service: PortableReplayService) -> APIRouter:
|
||||
def build_portable_replay_router(
|
||||
service: PortableReplayService,
|
||||
*,
|
||||
composition_runs: CompositionRunStore | None = None,
|
||||
queue: ObservatoryRecordedJobQueue | None = None,
|
||||
) -> APIRouter:
|
||||
router = APIRouter(tags=["observatory"])
|
||||
path = "/api/v1/observatory/portable-results/{result_id}/replays/{base_sha}/recording.rrd"
|
||||
|
||||
@@ -72,4 +90,85 @@ def build_portable_replay_router(service: PortableReplayService) -> APIRouter:
|
||||
},
|
||||
)
|
||||
|
||||
composition_path = (
|
||||
"/api/v1/observatory/ai-composition-runs/{run_id}/replays/{base_sha}/recording.rrd"
|
||||
)
|
||||
|
||||
def composition_members(run_id: str) -> tuple[str, ...]:
|
||||
if composition_runs is None or queue is None:
|
||||
raise HTTPException(503, "Составной просмотр AI-слоя недоступен.")
|
||||
try:
|
||||
run = composition_runs.get(run_id)
|
||||
jobs = tuple(queue.get(job_id) for job_id in run.job_ids)
|
||||
except (CompositionRunError, ObservatoryRecordedQueueError, ValueError) as exc:
|
||||
raise HTTPException(409, "Составной запуск AI-слоя недоступен.") from exc
|
||||
if not jobs or any(
|
||||
job.state != "succeeded"
|
||||
or job.publication_state != "published"
|
||||
or job.result_id is None
|
||||
for job in jobs
|
||||
):
|
||||
raise HTTPException(409, "Расчёт всех модулей этой конфигурации ещё не завершён.")
|
||||
return tuple(job.result_id for job in jobs if job.result_id is not None)
|
||||
|
||||
@router.head(composition_path)
|
||||
def prepare_composition(
|
||||
run_id: Annotated[str, Path(pattern=r"^ai-composition-[a-f0-9]{64}$")],
|
||||
base_sha: BaseSha,
|
||||
) -> Response:
|
||||
try:
|
||||
artifact = service.prepare_composition(run_id, composition_members(run_id), base_sha)
|
||||
except (
|
||||
ValueError,
|
||||
OSError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
CanonicalLabOverlayError,
|
||||
PortableResultViewError,
|
||||
) as exc:
|
||||
_LOG.exception("Composition replay packaging rejected")
|
||||
raise HTTPException(
|
||||
409, "Составной результат не удалось подготовить к просмотру."
|
||||
) from exc
|
||||
return Response(
|
||||
media_type="application/vnd.rerun.rrd",
|
||||
headers={
|
||||
"Content-Length": str(artifact.byte_length),
|
||||
"ETag": f'"{artifact.sha256}"',
|
||||
"X-Rerun-Format": "RRF2",
|
||||
"Cache-Control": "private, no-store",
|
||||
},
|
||||
)
|
||||
|
||||
@router.get(composition_path)
|
||||
def read_composition(
|
||||
run_id: Annotated[str, Path(pattern=r"^ai-composition-[a-f0-9]{64}$")],
|
||||
base_sha: BaseSha,
|
||||
generation: Annotated[str, Query(pattern=r"^[a-f0-9]{64}$")],
|
||||
) -> FileResponse:
|
||||
try:
|
||||
artifact = service.cached_composition(run_id, composition_members(run_id), base_sha)
|
||||
except (
|
||||
ValueError,
|
||||
OSError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
PortableResultViewError,
|
||||
) as exc:
|
||||
raise HTTPException(409, "Кэш составного результата не прошёл проверку.") from exc
|
||||
if artifact is None:
|
||||
raise HTTPException(409, "Составной просмотр ещё не подготовлен.")
|
||||
if artifact.sha256 != generation:
|
||||
raise HTTPException(412, "Версия составного просмотра изменилась.")
|
||||
return FileResponse(
|
||||
artifact.path,
|
||||
media_type="application/vnd.rerun.rrd",
|
||||
headers={
|
||||
"ETag": f'"{artifact.sha256}"',
|
||||
"X-Rerun-Format": "RRF2",
|
||||
"Cache-Control": "private, max-age=31536000, immutable",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
+117
-13
@@ -49,7 +49,10 @@ from k1link.viewer.recorded import (
|
||||
from k1link.viewer.recorded import (
|
||||
RecordedBlueprintError,
|
||||
recorded_blueprint_rrd,
|
||||
recorded_blueprint_sessions,
|
||||
)
|
||||
from k1link.viewer.recorded_blueprint_lifecycle import BlueprintSessionReleased
|
||||
from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit
|
||||
from k1link.viewer.rerun_bridge import RerunSceneSettings
|
||||
|
||||
DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
|
||||
@@ -102,7 +105,7 @@ class ReplayRequest(StrictApiModel):
|
||||
loop: bool = False
|
||||
|
||||
|
||||
class RecordedBlueprintRequest(StrictApiModel):
|
||||
class RecordedBlueprintIdentity(StrictApiModel):
|
||||
application_id: Literal["nodedc_mission_core_recorded"]
|
||||
recording_id: str = Field(
|
||||
min_length=1,
|
||||
@@ -114,25 +117,58 @@ class RecordedBlueprintRequest(StrictApiModel):
|
||||
max_length=32,
|
||||
pattern=r"^[a-f0-9]{32}$",
|
||||
)
|
||||
accumulation_seconds: float = Field(strict=True, ge=0.0, le=3600.0)
|
||||
|
||||
|
||||
class RecordedBlueprintLifecycleRequest(RecordedBlueprintIdentity):
|
||||
action: Literal["renew", "release"]
|
||||
|
||||
|
||||
EyeCoordinate = Annotated[float, Field(strict=True, allow_inf_nan=False)]
|
||||
EyeVector = tuple[EyeCoordinate, EyeCoordinate, EyeCoordinate]
|
||||
|
||||
|
||||
class RecordedBlueprintRequest(RecordedBlueprintIdentity):
|
||||
accumulation_seconds: float = Field(strict=True, ge=0.0, allow_inf_nan=False)
|
||||
show_points: StrictBool
|
||||
show_trajectory: StrictBool
|
||||
show_grid: StrictBool
|
||||
point_size: float = Field(default=2.5, strict=True, ge=0.1, le=32.0)
|
||||
point_size: float = Field(default=2.5, strict=True, ge=0.1, allow_inf_nan=False)
|
||||
color_mode: Literal["intensity", "height", "distance", "rgb", "class"] = "intensity"
|
||||
palette: Literal["turbo", "viridis", "plasma", "grayscale", "custom"] = "turbo"
|
||||
custom_color: str = Field(default="#f7f8f4", pattern=r"^#[0-9A-Fa-f]{6}$")
|
||||
active_view: Literal["spatial", "perception", "perception3d", "metrics"] = "spatial"
|
||||
view_reset_generation: Literal[0, 1] = 0
|
||||
unified_perception: StrictBool = False
|
||||
unified_camera_share: float = Field(strict=True, ge=0.1, le=0.9, default=0.46)
|
||||
semantic_layer: Literal["city", "vegetation"] | None = None
|
||||
plan_view: StrictBool = False
|
||||
show_detections_2d: StrictBool = False
|
||||
show_camera_image: StrictBool = True
|
||||
show_segmentation: StrictBool = False
|
||||
show_cuboids_3d: StrictBool = False
|
||||
show_costmap: StrictBool = False
|
||||
reactivate_updates: StrictBool = False
|
||||
follow_trajectory: StrictBool = False
|
||||
eye_position: EyeVector | None = None
|
||||
eye_look_target: EyeVector | None = None
|
||||
eye_up: EyeVector | None = None
|
||||
current_time_ns: int | None = Field(default=None, strict=True, ge=0, le=MAX_SAFE_INTEGER)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_eye_vectors(self) -> RecordedBlueprintRequest:
|
||||
vectors = (self.eye_position, self.eye_look_target, self.eye_up)
|
||||
if any(vector is None for vector in vectors):
|
||||
if not all(vector is None for vector in vectors):
|
||||
raise ValueError("all eye vectors must be supplied together")
|
||||
return self
|
||||
assert self.eye_position is not None
|
||||
assert self.eye_look_target is not None
|
||||
assert self.eye_up is not None
|
||||
if self.eye_position == self.eye_look_target:
|
||||
raise ValueError("eye position and look target must differ")
|
||||
if sum(value * value for value in self.eye_up) <= 1.0e-12:
|
||||
raise ValueError("eye up vector must be non-zero")
|
||||
return self
|
||||
|
||||
|
||||
class RecordedPerceptionRequest(StrictApiModel):
|
||||
@@ -365,6 +401,7 @@ def build_session_router(
|
||||
cursor: str | None = Query(default=None, max_length=128),
|
||||
scope: Literal["all", "source", "laboratory"] = "all",
|
||||
lab_contract: Literal["v1", "v2", "v3"] = "v1",
|
||||
pagination: Literal["cursor-v1"] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
_refresh_catalog(catalog_refresher)
|
||||
@@ -375,6 +412,14 @@ def build_session_router(
|
||||
include_capability_projections=lab_contract in ("v2", "v3"),
|
||||
)
|
||||
return {
|
||||
**(
|
||||
{
|
||||
"schema_version": "missioncore.observation-session-page/v1",
|
||||
"next_cursor": page.next_cursor,
|
||||
}
|
||||
if pagination == "cursor-v1"
|
||||
else {}
|
||||
),
|
||||
"items": [
|
||||
{
|
||||
"id": item.session_id,
|
||||
@@ -391,9 +436,7 @@ def build_session_router(
|
||||
else item.capture_attestation.as_dict()
|
||||
),
|
||||
**(
|
||||
{
|
||||
"lab": lab_catalog_document(item, lab_contract)
|
||||
}
|
||||
{"lab": lab_catalog_document(item, lab_contract)}
|
||||
if item.lab is not None
|
||||
else {}
|
||||
),
|
||||
@@ -411,7 +454,7 @@ def build_session_router(
|
||||
}
|
||||
for item in page.items
|
||||
if item.started_at_utc is not None
|
||||
]
|
||||
],
|
||||
}
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
@@ -870,9 +913,7 @@ def build_session_router(
|
||||
**response_kwargs,
|
||||
)
|
||||
|
||||
@router.get(
|
||||
"/api/v1/observation-sessions/{session_id}/canonical-lab/spatial-frame"
|
||||
)
|
||||
@router.get("/api/v1/observation-sessions/{session_id}/canonical-lab/spatial-frame")
|
||||
async def get_observation_session_canonical_lab_spatial_frame(
|
||||
session_id: str,
|
||||
generation: Annotated[str, Query(min_length=64, max_length=64)],
|
||||
@@ -937,20 +978,37 @@ def build_session_router(
|
||||
headers={
|
||||
"Cache-Control": "private, max-age=31536000, immutable",
|
||||
"ETag": (
|
||||
f'"{generation}:{CANONICAL_LAB_SPATIAL_PROFILE}:'
|
||||
f'{payload["source_time_ns"]}"'
|
||||
f'"{generation}:{CANONICAL_LAB_SPATIAL_PROFILE}:{payload["source_time_ns"]}"'
|
||||
),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
@router.post("/api/v1/observation-sessions/{session_id}/blueprint-lifecycle")
|
||||
async def update_recorded_blueprint_lifecycle(
|
||||
session_id: str,
|
||||
request: RecordedBlueprintLifecycleRequest,
|
||||
) -> Response:
|
||||
# Releasing an ephemeral owner must still work after source removal.
|
||||
# The random viewport identity authorizes only its own memory resource;
|
||||
# this route never materializes or deletes recordings/artifacts.
|
||||
if not SAFE_SOURCE_ID.fullmatch(session_id):
|
||||
raise HTTPException(status_code=422, detail="Некорректный идентификатор сессии.")
|
||||
key = (request.application_id, request.recording_id, request.blueprint_session_id)
|
||||
if request.action == "release":
|
||||
await run_in_threadpool(recorded_blueprint_sessions.release, key)
|
||||
else:
|
||||
await run_in_threadpool(recorded_blueprint_sessions.renew, key)
|
||||
return Response(status_code=204, headers={"Cache-Control": "no-store"})
|
||||
|
||||
@router.post("/api/v1/observation-sessions/{session_id}/blueprint.rrd")
|
||||
async def get_observation_session_blueprint(
|
||||
session_id: str,
|
||||
request: RecordedBlueprintRequest,
|
||||
) -> Response:
|
||||
camera_max_orbital_radius: float | None = None
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
command = await run_in_threadpool(
|
||||
_prepare_replay,
|
||||
store,
|
||||
catalog_refresher,
|
||||
@@ -977,19 +1035,55 @@ def build_session_router(
|
||||
active_view=request.active_view,
|
||||
view_reset_generation=request.view_reset_generation,
|
||||
unified_perception=request.unified_perception,
|
||||
unified_camera_share=request.unified_camera_share,
|
||||
semantic_layer=request.semantic_layer,
|
||||
plan_view=request.plan_view,
|
||||
show_detections_2d=request.show_detections_2d,
|
||||
show_camera_image=request.show_camera_image,
|
||||
show_segmentation=request.show_segmentation,
|
||||
show_cuboids_3d=request.show_cuboids_3d,
|
||||
show_costmap=request.show_costmap,
|
||||
reactivate_updates=request.reactivate_updates,
|
||||
follow_trajectory=request.follow_trajectory,
|
||||
eye_position=request.eye_position,
|
||||
eye_look_target=request.eye_look_target,
|
||||
eye_up=request.eye_up,
|
||||
)
|
||||
if request.current_time_ns is not None:
|
||||
camera_recording = None
|
||||
if recording_preparation_manager is not None:
|
||||
snapshot = recording_preparation_manager.status(session_id)
|
||||
if (
|
||||
snapshot is not None
|
||||
and snapshot.state == "ready"
|
||||
and snapshot.recording is not None
|
||||
):
|
||||
camera_recording = snapshot.recording
|
||||
# A composition replay consumes the immutable base launch but
|
||||
# does not GET its recording. Its short launch reservation can
|
||||
# therefore expire while the combined RRD remains open. Restore
|
||||
# the already-published base descriptor with bounded stat checks
|
||||
# so later layer toggles still receive the native zoom limit.
|
||||
if camera_recording is None and recording_materializer is not None:
|
||||
camera_recording = await run_in_threadpool(
|
||||
recording_materializer.restore_published,
|
||||
command,
|
||||
)
|
||||
if camera_recording is not None:
|
||||
camera_max_orbital_radius = await run_in_threadpool(
|
||||
recorded_orbital_radius_limit,
|
||||
camera_recording.path,
|
||||
current_time_ns=request.current_time_ns,
|
||||
accumulation_seconds=request.accumulation_seconds,
|
||||
show_points=request.show_points,
|
||||
show_trajectory=request.show_trajectory,
|
||||
)
|
||||
except SessionNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except (SessionNotReplayableError, SessionIntegrityError) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except BlueprintSessionReleased as exc:
|
||||
raise HTTPException(status_code=410, detail="Сессия визуализатора закрыта.") from exc
|
||||
except RecordedBlueprintError as exc:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
@@ -1007,6 +1101,16 @@ def build_session_router(
|
||||
"Cache-Control": "no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Content-Disposition": 'inline; filename="blueprint.rrd"',
|
||||
**(
|
||||
{
|
||||
"X-MissionCore-Camera-Max-Orbital-Radius": format(
|
||||
camera_max_orbital_radius,
|
||||
".9g",
|
||||
)
|
||||
}
|
||||
if camera_max_orbital_radius is not None
|
||||
else {}
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user