feat(observatory): ship modular AI inference labs

This commit is contained in:
DCCONSTRUCTIONS
2026-09-04 17:59:05 +03:00
parent eff60e490a
commit cada687173
145 changed files with 17651 additions and 1667 deletions
+211
View File
@@ -0,0 +1,211 @@
"""Reuse exact v2 inputs before decoding the source, without changing v2 identity.
The legacy producer hashes its own file into every pack. Keep that producer
unchanged: this adapter only selects and verifies an existing pack, or calls
the original builder. A cache hit is not a streaming/cold-start qualification.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
import re
import stat
from pathlib import Path
from typing import Any
from . import lidar_replay
from .lidar_contract import K1_LIDAR_PACK_V2_PROFILE
from .lidar_replay import (
DEFAULT_POSE_COVERAGE_THRESHOLD_MS,
LIDAR_MANIFEST_NAME,
LIDAR_REPLAY_PACK_SCHEMA,
LidarReplayError,
LidarReplayPackV2,
)
_PACK_ID = re.compile(r"^lidar-replay-pack-[a-f0-9]{64}$")
_SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_MAX_MANIFEST_BYTES = 128 * 1024
_HASH_CHUNK_BYTES = 1024 * 1024
type _FileStamp = tuple[int, int, int, int, int]
def prepare_lidar_replay_pack_v2(
capture_path: Path,
output_root: Path,
*,
session_id: str | None = None,
pose_coverage_threshold_ms: float = DEFAULT_POSE_COVERAGE_THRESHOLD_MS,
) -> Path:
"""Return the exact validated input, skipping source decode on a cache hit.
Only current-producer packs with exact raw/metadata/clock-origin digests may
be reused. Their ordinary strict reader still checks artifacts, arrays,
logical content and equivalence. Corruption fails without replacing evidence.
No source or pack array survives this call.
"""
source = capture_path.expanduser().resolve(strict=True)
if source.name != "mqtt.raw.k1mqtt" or not source.is_file():
raise LidarReplayError("LiDAR replay source must be mqtt.raw.k1mqtt")
metadata = source.with_name("mqtt.metadata.jsonl")
if not metadata.is_file():
raise LidarReplayError("exact host timing requires mqtt.metadata.jsonl")
if (
not math.isfinite(pose_coverage_threshold_ms)
or not 0 < pose_coverage_threshold_ms <= 10_000
):
raise LidarReplayError("pose coverage threshold is invalid")
resolved_session = session_id or source.parents[2].name
if _SESSION_ID.fullmatch(resolved_session) is None:
raise LidarReplayError("LiDAR replay session id is unsafe")
parent = output_root.expanduser().absolute()
if parent.is_symlink():
raise LidarReplayError("LiDAR preparation cache cannot be a symlink")
producer = Path(lidar_replay.__file__).resolve(strict=True)
producer_sha256, _ = _hash_regular_file(producer)
candidates = _candidates(parent, resolved_session, producer_sha256)
# A cold directory does not add another whole-source hash pass.
if not candidates:
return lidar_replay.build_lidar_replay_pack_v2(
source,
parent,
session_id=resolved_session,
pose_coverage_threshold_ms=pose_coverage_threshold_ms,
)
evidence, stamps = _source_evidence(source, metadata)
matches = [
(root, identity)
for root, identity in candidates
if identity.get("source_evidence") == evidence
]
if len(matches) > 1:
raise LidarReplayError("LiDAR preparation cache has ambiguous source identity")
if not matches:
result = lidar_replay.build_lidar_replay_pack_v2(
source,
parent,
session_id=resolved_session,
pose_coverage_threshold_ms=pose_coverage_threshold_ms,
)
_check_source_stamps(source, stamps)
return result
root, identity = matches[0]
pack = LidarReplayPackV2(root)
try:
if pack.identity != identity:
raise LidarReplayError("LiDAR preparation cache changed during validation")
pose_binding = pack.quality.get("pose_binding")
if (
not isinstance(pose_binding, dict)
or pose_binding.get("threshold_ms") != pose_coverage_threshold_ms
):
# v2 did not include this report parameter in its identity. Never
# silently return another report or overwrite the existing pack.
raise LidarReplayError("LiDAR cached pose coverage threshold differs")
_check_source_stamps(source, stamps)
return root
finally:
pack.close()
def _candidates(
parent: Path,
session_id: str,
producer_sha256: str,
) -> list[tuple[Path, dict[str, Any]]]:
if not parent.exists():
return []
result: list[tuple[Path, dict[str, Any]]] = []
for root in parent.iterdir():
if _PACK_ID.fullmatch(root.name) is None:
continue
if root.is_symlink() or not root.is_dir():
raise LidarReplayError("LiDAR preparation cache entry is unsafe")
manifest_path = root / LIDAR_MANIFEST_NAME
if manifest_path.is_symlink():
raise LidarReplayError("LiDAR preparation manifest cannot be a symlink")
try:
with manifest_path.open("rb") as stream:
payload = stream.read(_MAX_MANIFEST_BYTES + 1)
if len(payload) > _MAX_MANIFEST_BYTES:
raise ValueError("manifest too large")
manifest = json.loads(payload)
except (OSError, UnicodeDecodeError, ValueError) as exc:
raise LidarReplayError("LiDAR preparation manifest is invalid") from exc
identity = manifest.get("identity") if isinstance(manifest, dict) else None
if not isinstance(identity, dict):
raise LidarReplayError("LiDAR preparation identity is invalid")
# Unrelated profiles/producers are preserved, never eagerly decoded.
if (
identity.get("session_id") != session_id
or identity.get("producer_sha256") != producer_sha256
):
continue
encoded = json.dumps(
identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False
).encode()
digest = hashlib.sha256(encoded).hexdigest()
if (
manifest.get("schema_version") != LIDAR_REPLAY_PACK_SCHEMA
or identity.get("schema_version") != LIDAR_REPLAY_PACK_SCHEMA
or manifest.get("pack_id") != root.name
or root.name != f"lidar-replay-pack-{digest}"
or manifest.get("identity_sha256") != digest
or identity.get("lidar_evidence_profile") != K1_LIDAR_PACK_V2_PROFILE.to_dict()
):
raise LidarReplayError("LiDAR preparation identity changed")
result.append((root.resolve(strict=True), identity))
return result
def _stamp(value: os.stat_result) -> _FileStamp:
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns
def _hash_regular_file(path: Path) -> tuple[str, _FileStamp]:
before = path.lstat()
if not stat.S_ISREG(before.st_mode):
raise LidarReplayError("LiDAR source evidence must be a regular file")
digest = hashlib.sha256()
with path.open("rb") as stream:
if _stamp(os.fstat(stream.fileno())) != _stamp(before):
raise LidarReplayError("LiDAR source evidence changed before hashing")
while chunk := stream.read(_HASH_CHUNK_BYTES):
digest.update(chunk)
if _stamp(os.fstat(stream.fileno())) != _stamp(before):
raise LidarReplayError("LiDAR source evidence changed during hashing")
if _stamp(path.lstat()) != _stamp(before):
raise LidarReplayError("LiDAR source evidence changed after hashing")
return digest.hexdigest(), _stamp(before)
def _source_evidence(
source: Path,
metadata: Path,
) -> tuple[dict[str, object], dict[Path, _FileStamp]]:
paths = {"raw": source, "metadata": metadata}
origin = source.with_name("mqtt.timeline.origin.json")
if origin.exists():
paths["clock_origin"] = origin
evidence: dict[str, object] = {}
stamps: dict[Path, _FileStamp] = {}
for role, path in paths.items():
digest, stamp = _hash_regular_file(path)
evidence[role] = {"sha256": digest, "byte_length": stamp[2]}
stamps[path] = stamp
return evidence, stamps
def _check_source_stamps(source: Path, stamps: dict[Path, _FileStamp]) -> None:
origin = source.with_name("mqtt.timeline.origin.json")
if origin.exists() != (origin in stamps):
raise LidarReplayError("LiDAR source clock origin changed during preparation")
for path, expected in stamps.items():
if not stat.S_ISREG(path.lstat().st_mode) or _stamp(path.lstat()) != expected:
raise LidarReplayError("LiDAR source evidence changed during preparation")
@@ -36,6 +36,10 @@ APPLICATION_ID: Final = "nodedc_mission_core_recorded"
SESSION_TIMELINE: Final = "session_time"
RENDERER_VERSION: Final = "upstream-rerun-0.36.3-encoded-optimized-v6"
REPLAY_RENDERER_VERSION: Final = "upstream-rerun-0.36.3-canonical-replay-v1"
CANONICAL_REPLAY_RESULT_ID: Final = re.compile(
r"^(?:lab-v1-vegetation-shadow|m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance)|ai-composition)-[a-f0-9]{64}$"
)
MAX_SOURCE_BYTES: Final = 768 * 1024 * 1024
MAX_OVERLAY_BYTES: Final = 256 * 1024 * 1024
MAX_REPLAY_BYTES: Final = 1024 * 1024 * 1024
@@ -138,9 +142,7 @@ class CanonicalLabReplayArtifact:
_render_lock = threading.Lock()
_memory_cache: dict[tuple[str, str, str], CanonicalLabOverlayArtifact] = {}
_replay_lock = threading.Lock()
_replay_memory_cache: dict[
tuple[str, str, str, str], CanonicalLabReplayArtifact
] = {}
_replay_memory_cache: dict[tuple[str, str, str, str], CanonicalLabReplayArtifact] = {}
def canonical_recording_id(path: Path) -> str:
@@ -283,9 +285,7 @@ def canonical_lab_replay(
or not _is_sha256(base_generation_sha256)
or _sha256(base) != base_generation_sha256
or not _artifact_is_regular(overlay)
or re.fullmatch(
r"(?:lab-v1-vegetation-shadow|m49-tgs-portable-review)-[a-f0-9]{64}", result_id
) is None
or CANONICAL_REPLAY_RESULT_ID.fullmatch(result_id) is None
or not recording_id
or len(recording_id) > 128
):
@@ -447,12 +447,11 @@ def _verified_camera_source(root: Path, route: dict[str, Any], jobs_root: Path)
or not isinstance(files, list)
):
raise CanonicalLabOverlayError("camera job source contract changed")
epoch_prefix = PurePosixPath(
"input/camera/sensor.camera.right"
) / f"epoch-{source.get('codec_epoch')}"
epoch_prefix = (
PurePosixPath("input/camera/sensor.camera.right") / f"epoch-{source.get('codec_epoch')}"
)
required = [epoch_prefix / "init.mp4"] + [
epoch_prefix / "segments" / f"{index}.m4s"
for index in range(1, frame_count + 1)
epoch_prefix / "segments" / f"{index}.m4s" for index in range(1, frame_count + 1)
]
descriptors = {
item.get("path"): item
@@ -609,10 +608,7 @@ def _render_overlay(
"/perception/camera/image",
rr.VideoFrameReference(nanoseconds=int(video_references[index])),
)
masks = {
layer_id: _read_mask(archive, index)
for layer_id, archive in archives.items()
}
masks = {layer_id: _read_mask(archive, index) for layer_id, archive in archives.items()}
for layer_id, mask in masks.items():
recording.log(
f"/perception/camera/segmentation/{layer_id}",
@@ -731,15 +727,17 @@ def _video_reference_timestamps(
if (
len(video_timestamps) < int(len(frame_times) * 0.9)
or np.any(np.diff(video_timestamps) < 0)
or abs(int(video_timestamps[-1]) - int(relative_frame_times[-1]))
> 2_000_000_000
or abs(int(video_timestamps[-1]) - int(relative_frame_times[-1])) > 2_000_000_000
):
raise CanonicalLabOverlayError("video proxy timeline changed")
indices = np.searchsorted(
video_timestamps,
relative_frame_times,
side="right",
) - 1
indices = (
np.searchsorted(
video_timestamps,
relative_frame_times,
side="right",
)
- 1
)
return video_timestamps[np.clip(indices, 0, len(video_timestamps) - 1)]
@@ -770,9 +768,7 @@ def _semantic_palette(classes: list[object]) -> tuple[int, ...]:
or not isinstance(color, list)
or len(color) != 3
or any(
not isinstance(channel, int)
or isinstance(channel, bool)
or not 0 <= channel <= 255
not isinstance(channel, int) or isinstance(channel, bool) or not 0 <= channel <= 255
for channel in color
)
):
@@ -820,9 +816,7 @@ def semantic_component_boxes(
mask, class_id, minimum_pixels=minimum_pixels
)[:12]:
score = min(0.99, 0.5 + pixels / 20_000)
candidates.append(
(score, [left, top, right, bottom], f"{label} · {score:.0%}")
)
candidates.append((score, [left, top, right, bottom], f"{label} · {score:.0%}"))
candidates.sort(key=lambda row: (-row[0], row[1][1], row[1][0]))
selected = candidates[:32]
return [row[1] for row in selected], [row[2] for row in selected]
@@ -1001,6 +995,8 @@ def _sha256(path: Path) -> str:
def _is_sha256(value: object) -> bool:
return isinstance(value, str) and len(value) == 64 and all(
character in "0123456789abcdef" for character in value
return (
isinstance(value, str)
and len(value) == 64
and all(character in "0123456789abcdef" for character in value)
)
+282
View File
@@ -0,0 +1,282 @@
"""Append-only bindings from one operator composition submission to its jobs."""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Final, cast
from k1link.observatory.modular_composition import CompositionSpec, canonical_bytes
RUN_SCHEMA: Final = "missioncore.observatory-ai-composition-run/v1"
RUN_PROJECTION_SCHEMA: Final = "missioncore.observatory-ai-composition-run-projection/v1"
_RUN = re.compile(r"ai-composition-[a-f0-9]{64}\Z")
_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,159}\Z")
class CompositionRunError(ValueError):
"""A composition-run binding is invalid or changed after admission."""
@dataclass(frozen=True, slots=True)
class CompositionRun:
run_id: str
source_session_id: str
composition_sha256: str
module_ids: tuple[str, ...]
setup_ids: tuple[str, ...]
job_ids: tuple[str, ...]
created_at_utc: str
def __post_init__(self) -> None:
if _RUN.fullmatch(self.run_id) is None:
raise CompositionRunError("invalid composition run id")
if any(
_ID.fullmatch(value) is None
for value in (
self.source_session_id,
*self.module_ids,
*self.setup_ids,
*self.job_ids,
)
):
raise CompositionRunError("invalid composition run identity")
if (
not re.fullmatch(r"[a-f0-9]{64}", self.composition_sha256)
or not self.module_ids
or len(self.setup_ids) != len(self.job_ids)
or len(set(self.setup_ids)) != len(self.setup_ids)
or len(set(self.job_ids)) != len(self.job_ids)
):
raise CompositionRunError("invalid composition run members")
if not self.created_at_utc.endswith("Z"):
raise CompositionRunError("composition run timestamp must be UTC")
def as_dict(self) -> dict[str, object]:
return {
"schema_version": RUN_SCHEMA,
"run_id": self.run_id,
"source_session_id": self.source_session_id,
"composition_sha256": self.composition_sha256,
"module_ids": list(self.module_ids),
"setup_ids": list(self.setup_ids),
"job_ids": list(self.job_ids),
"created_at_utc": self.created_at_utc,
}
class CompositionRunStore:
def __init__(self, root: Path) -> None:
root = root.expanduser().absolute()
root.mkdir(parents=True, exist_ok=True, mode=0o700)
if root.is_symlink() or root.resolve() != root:
raise CompositionRunError("composition run store must be a real directory")
self.root = root
def save(
self,
*,
source_session_id: str,
composition: CompositionSpec,
setup_ids: tuple[str, ...],
job_ids: tuple[str, ...],
idempotency_key: str,
created_at_utc: str,
) -> CompositionRun:
digest = hashlib.sha256(
canonical_bytes(
{
"source_session_id": source_session_id,
"composition_sha256": composition.sha256,
"idempotency_key": idempotency_key,
}
)
).hexdigest()
run = CompositionRun(
run_id=f"ai-composition-{digest}",
source_session_id=source_session_id,
composition_sha256=composition.sha256,
module_ids=tuple(
node.module.module_id
for node in composition.nodes
if node.module.group != "preparation"
),
setup_ids=setup_ids,
job_ids=job_ids,
created_at_utc=created_at_utc,
)
destination = self.root / f"{run.run_id}.json"
payload = canonical_bytes(run.as_dict())
if destination.exists():
if destination.is_symlink() or destination.read_bytes() != payload:
raise CompositionRunError("composition run identity changed")
return run
descriptor, temporary = tempfile.mkstemp(prefix=".run-", dir=self.root)
path = Path(temporary)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
path.chmod(0o444)
path.rename(destination)
finally:
path.unlink(missing_ok=True)
return run
def get(self, run_id: str) -> CompositionRun:
if _RUN.fullmatch(run_id) is None:
raise CompositionRunError("invalid composition run id")
path = self.root / f"{run_id}.json"
if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024:
raise CompositionRunError("composition run is unavailable")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise CompositionRunError("composition run is unreadable") from exc
return _decode(value)
def list(
self,
*,
source_session_id: str | None = None,
include_hidden: bool = True,
) -> tuple[CompositionRun, ...]:
values: list[CompositionRun] = []
for path in self.root.glob("ai-composition-*.json"):
if _RUN.fullmatch(path.stem) is None:
continue
run = self.get(path.stem)
if (source_session_id is None or run.source_session_id == source_session_id) and (
include_hidden or self.is_visible(run.run_id)
):
values.append(run)
return tuple(sorted(values, key=lambda row: (row.created_at_utc, row.run_id), reverse=True))
def display_name(self, run_id: str) -> str | None:
projection = self._projection(run_id)
value = projection.get("display_name")
return cast(str, value) if isinstance(value, str) else None
def is_visible(self, run_id: str) -> bool:
return self._projection(run_id).get("visible") is not False
def rename_projection(self, run_id: str, display_name: str) -> str:
self.get(run_id)
normalized = display_name.strip()
if not normalized or len(normalized) > 160:
raise CompositionRunError("invalid composition run display name")
projection = self._projection(run_id)
self._write_projection(
run_id, display_name=normalized, visible=projection.get("visible") is not False
)
return normalized
def delete_projection(self, run_id: str) -> None:
self.get(run_id)
projection = self._projection(run_id)
self._write_projection(
run_id,
display_name=cast(str | None, projection.get("display_name")),
visible=False,
)
def _projection(self, run_id: str) -> dict[str, object]:
if _RUN.fullmatch(run_id) is None:
raise CompositionRunError("invalid composition run id")
path = self.root / f"{run_id}.projection.json"
if not path.exists():
return {"display_name": None, "visible": True}
if path.is_symlink() or not path.is_file() or path.stat().st_size > 4 * 1024:
raise CompositionRunError("composition run projection is unavailable")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise CompositionRunError("composition run projection is unreadable") from exc
expected = {"schema_version", "run_id", "display_name", "visible"}
if (
not isinstance(value, dict)
or set(value) != expected
or value.get("schema_version") != RUN_PROJECTION_SCHEMA
or value.get("run_id") != run_id
or not isinstance(value.get("visible"), bool)
or (
value.get("display_name") is not None
and (
not isinstance(value.get("display_name"), str)
or not cast(str, value["display_name"]).strip()
or cast(str, value["display_name"]).strip() != value["display_name"]
or len(cast(str, value["display_name"])) > 160
)
)
):
raise CompositionRunError("invalid composition run projection")
return cast(dict[str, object], value)
def _write_projection(
self,
run_id: str,
*,
display_name: str | None,
visible: bool,
) -> None:
destination = self.root / f"{run_id}.projection.json"
payload = canonical_bytes(
{
"schema_version": RUN_PROJECTION_SCHEMA,
"run_id": run_id,
"display_name": display_name,
"visible": visible,
}
)
descriptor, temporary = tempfile.mkstemp(prefix=".projection-", dir=self.root)
path = Path(temporary)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
path.chmod(0o600)
path.replace(destination)
finally:
path.unlink(missing_ok=True)
def _decode(value: object) -> CompositionRun:
keys = {
"schema_version",
"run_id",
"source_session_id",
"composition_sha256",
"module_ids",
"setup_ids",
"job_ids",
"created_at_utc",
}
if (
not isinstance(value, dict)
or set(value) != keys
or value.get("schema_version") != RUN_SCHEMA
):
raise CompositionRunError("invalid composition run document")
row = cast(dict[str, object], value)
arrays = (row["module_ids"], row["setup_ids"], row["job_ids"])
if any(
not isinstance(items, list) or any(not isinstance(item, str) for item in items)
for items in arrays
):
raise CompositionRunError("invalid composition run member arrays")
return CompositionRun(
run_id=cast(str, row["run_id"]),
source_session_id=cast(str, row["source_session_id"]),
composition_sha256=cast(str, row["composition_sha256"]),
module_ids=tuple(cast(list[str], row["module_ids"])),
setup_ids=tuple(cast(list[str], row["setup_ids"])),
job_ids=tuple(cast(list[str], row["job_ids"])),
created_at_utc=cast(str, row["created_at_utc"]),
)
+265
View File
@@ -0,0 +1,265 @@
"""Strict local Observatory ontology shared by planning, publication and replay UI."""
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Final, cast
from k1link.observatory.modular_composition import CompositionSpec
ONTOLOGY_SCHEMA: Final = "missioncore.observatory-domain-ontology/v1"
_ID = re.compile(r"[a-z][a-z0-9._-]{1,95}\Z")
_FIELD_ID = re.compile(r"[a-z][a-z0-9_]{1,63}\Z")
_CARDINALITIES: Final = {
"one-to-one",
"one-to-many",
"many-to-one",
"many-to-many",
"one-to-zero-or-one",
"many-to-zero-or-one",
}
class ObservatoryOntologyError(ValueError):
"""The local ontology cannot answer a product query exactly."""
@dataclass(frozen=True, slots=True)
class ViewerLayer:
layer_id: str
pane_id: str
label: str
control: str
order: int
def as_dict(self) -> dict[str, object]:
return {
"layer_id": self.layer_id,
"pane_id": self.pane_id,
"label": self.label,
"control": self.control,
"order": self.order,
}
@dataclass(frozen=True, slots=True)
class ModuleProjection:
module_id: str
configuration_label: str
layer_ids: tuple[str, ...]
class ObservatoryDomainOntology:
"""Versioned named-query projection; no graph service or Platform dependency."""
def __init__(
self,
*,
document: dict[str, object],
layers: tuple[ViewerLayer, ...],
modules: tuple[ModuleProjection, ...],
) -> None:
self.document = document
self.layers = layers
self._layers = {layer.layer_id: layer for layer in layers}
self._modules = {module.module_id: module for module in modules}
self._module_order = {module.module_id: order for order, module in enumerate(modules)}
if len(self._layers) != len(layers) or len(self._modules) != len(modules):
raise ObservatoryOntologyError("ontology identities must be unique")
for module in modules:
if any(layer not in self._layers for layer in module.layer_ids):
raise ObservatoryOntologyError("module references an unknown viewer layer")
@classmethod
def from_file(cls, path: Path) -> ObservatoryDomainOntology:
candidate = path.expanduser().absolute()
if (
candidate.is_symlink()
or not candidate.is_file()
or candidate.stat().st_size > 1024 * 1024
):
raise ObservatoryOntologyError("ontology must be a bounded regular file")
try:
value = json.loads(candidate.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ObservatoryOntologyError("ontology is unreadable") from exc
if not isinstance(value, dict) or value.get("schema_version") != ONTOLOGY_SCHEMA:
raise ObservatoryOntologyError("unsupported Observatory ontology")
required = {
"schema_version",
"ontology_id",
"version",
"owner",
"lifecycle",
"entities",
"relations",
"panes",
"layers",
"module_projections",
"named_queries",
"platform_sync",
}
if set(value) != required:
raise ObservatoryOntologyError("unexpected Observatory ontology fields")
entity_ids = _validate_entities(value["entities"])
_validate_relations(value["relations"], entity_ids)
pane_ids = _validate_panes(value["panes"])
layers = tuple(_layer(item) for item in _array(value["layers"], "layers"))
if len({layer.layer_id for layer in layers}) != len(layers):
raise ObservatoryOntologyError("ontology layer identities must be unique")
if any(layer.pane_id not in pane_ids for layer in layers):
raise ObservatoryOntologyError("viewer layer references an unknown pane")
modules = tuple(_module(item) for item in _array(value["module_projections"], "modules"))
named = value["named_queries"]
if named != [
"recording.capture-context",
"composition.configuration-label",
"composition.member-results",
"composition.viewer-layers",
"composition.ready-state",
"lab.view-profile",
]:
raise ObservatoryOntologyError("required named queries are absent")
return cls(document=cast(dict[str, object], value), layers=layers, modules=modules)
def project_module_ids(self, module_ids: tuple[str, ...]) -> dict[str, object]:
selected = []
layer_ids: set[str] = set()
for module_id in module_ids:
projection = self._modules.get(module_id)
if projection is None:
raise ObservatoryOntologyError(f"module {module_id} has no ontology projection")
selected.append(projection)
layer_ids.update(projection.layer_ids)
selected.sort(key=lambda row: self._module_order[row.module_id])
layers = sorted(
(self._layers[layer_id] for layer_id in layer_ids),
key=lambda row: (row.pane_id, row.order),
)
return {
"schema_version": "missioncore.observatory-presentation-projection/v1",
"modules": [
{"module_id": row.module_id, "label": row.configuration_label} for row in selected
],
"configuration_label": " · ".join(row.configuration_label for row in selected),
"viewer_layers": [row.as_dict() for row in layers],
}
def project_composition(self, composition: CompositionSpec) -> dict[str, object]:
return self.project_module_ids(
tuple(
node.module.module_id
for node in composition.nodes
if node.module.group != "preparation"
)
)
def _array(value: object, label: str) -> list[object]:
if not isinstance(value, list):
raise ObservatoryOntologyError(f"ontology {label} must be an array")
return value
def _record(value: object, keys: set[str], label: str) -> dict[str, object]:
if not isinstance(value, dict) or set(value) != keys:
raise ObservatoryOntologyError(f"invalid ontology {label}")
return cast(dict[str, object], value)
def _identifier(value: object, label: str) -> str:
if not isinstance(value, str) or not _ID.fullmatch(value):
raise ObservatoryOntologyError(f"invalid {label}")
return value
def _nonempty_text(value: object, label: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ObservatoryOntologyError(f"invalid {label}")
return value
def _validate_entities(value: object) -> set[str]:
identities: set[str] = set()
for item in _array(value, "entities"):
row = _record(item, {"id", "identity", "owner", "lifecycle"}, "entity")
entity_id = _identifier(row["id"], "entity id")
if entity_id in identities:
raise ObservatoryOntologyError("ontology entity identities must be unique")
identities.add(entity_id)
identity_field = row["identity"]
if not isinstance(identity_field, str) or not _FIELD_ID.fullmatch(identity_field):
raise ObservatoryOntologyError("invalid entity identity field")
_nonempty_text(row["owner"], "entity owner")
_nonempty_text(row["lifecycle"], "entity lifecycle")
if not identities:
raise ObservatoryOntologyError("ontology entities must not be empty")
return identities
def _validate_relations(value: object, entity_ids: set[str]) -> None:
identities: set[str] = set()
for item in _array(value, "relations"):
row = _record(item, {"id", "from", "to", "cardinality"}, "relation")
relation_id = _identifier(row["id"], "relation id")
if relation_id in identities:
raise ObservatoryOntologyError("ontology relation identities must be unique")
identities.add(relation_id)
source = _identifier(row["from"], "relation source")
target = _identifier(row["to"], "relation target")
if source not in entity_ids or target not in entity_ids:
raise ObservatoryOntologyError("relation references an unknown entity")
if row["cardinality"] not in _CARDINALITIES:
raise ObservatoryOntologyError("invalid relation cardinality")
def _validate_panes(value: object) -> set[str]:
identities: set[str] = set()
for item in _array(value, "panes"):
row = _record(item, {"pane_id", "label", "order"}, "pane")
pane_id = _identifier(row["pane_id"], "pane id")
if pane_id in identities:
raise ObservatoryOntologyError("ontology pane identities must be unique")
identities.add(pane_id)
_nonempty_text(row["label"], "pane label")
if not isinstance(row["order"], int) or isinstance(row["order"], bool):
raise ObservatoryOntologyError("invalid pane order")
return identities
def _layer(value: object) -> ViewerLayer:
row = _record(value, {"layer_id", "pane_id", "label", "control", "order"}, "layer")
if (
not isinstance(row["label"], str)
or not row["label"]
or row["control"] not in {"toggle", "toggle-with-settings"}
or not isinstance(row["order"], int)
or isinstance(row["order"], bool)
):
raise ObservatoryOntologyError("invalid viewer layer presentation")
return ViewerLayer(
_identifier(row["layer_id"], "layer id"),
_identifier(row["pane_id"], "pane id"),
row["label"],
cast(str, row["control"]),
row["order"],
)
def _module(value: object) -> ModuleProjection:
row = _record(value, {"module_id", "configuration_label", "layers"}, "module projection")
if not isinstance(row["configuration_label"], str) or not row["configuration_label"]:
raise ObservatoryOntologyError("invalid module configuration label")
layers = tuple(
_identifier(item, "viewer layer id") for item in _array(row["layers"], "module layers")
)
if len(layers) != len(set(layers)):
raise ObservatoryOntologyError("module viewer layers must be unique")
return ModuleProjection(
_identifier(row["module_id"], "module id"),
row["configuration_label"],
layers,
)
@@ -160,8 +160,10 @@ class InstalledLabDockerLaunch:
"com.nodedc.definition-sha256",
"com.nodedc.job-id",
"com.nodedc.managed-by",
"com.nodedc.module-id",
"com.nodedc.package-sha256",
"com.nodedc.product",
"com.nodedc.role",
"com.nodedc.stack",
}
if set(self.labels) != required_labels:
@@ -170,7 +172,9 @@ class InstalledLabDockerLaunch:
self.labels["com.nodedc.authority"] != "observation-only"
or self.labels["com.nodedc.component"] != self.container.container_id
or self.labels["com.nodedc.managed-by"] != "mission-core-worker"
or self.labels["com.nodedc.module-id"] != self.container.container_id
or self.labels["com.nodedc.product"] != "mission-core"
or self.labels["com.nodedc.role"] != "ai-module"
or self.labels["com.nodedc.stack"] != "observatory"
):
raise InstalledLabPackageRunnerError("Docker launch labels changed")
@@ -251,14 +255,10 @@ class DockerEngineInstalledLabLauncher:
or len(image_sha256s) != len(set(image_sha256s))
or any(_SHA256.fullmatch(value) is None for value in image_sha256s)
):
raise InstalledLabPackageRunnerError(
"installed LAB image inventory is invalid"
)
raise InstalledLabPackageRunnerError("installed LAB image inventory is invalid")
if self.transport_factory is None:
_require_local_socket(self.socket_path)
transport: httpx.BaseTransport = httpx.HTTPTransport(
uds=str(self.socket_path)
)
transport: httpx.BaseTransport = httpx.HTTPTransport(uds=str(self.socket_path))
else:
transport = self.transport_factory()
with httpx.Client(
@@ -282,7 +282,7 @@ class DockerEngineInstalledLabLauncher:
def _create(self, client: httpx.Client, launch: InstalledLabDockerLaunch) -> str:
component = launch.container.container_id[:32]
name = f"ndc-observatory-{component}-{launch.name_token}"
name = f"ndc-mission-core-ai-module-{component}-{launch.name_token}"
response = self._response(
client,
"POST",
@@ -455,7 +455,9 @@ class InstalledLabPackageProfileRunner:
output_root.mkdir(mode=0o700)
steps_root = job_root / "steps"
steps_root.mkdir(mode=0o700)
plan_path = job_root / "run-plan.json"
plan_root = job_root / "plan"
plan_root.mkdir(mode=0o700)
plan_path = plan_root / "run-plan.json"
_write_exclusive(
plan_path,
canonical_json(
@@ -530,11 +532,11 @@ class InstalledLabPackageProfileRunner:
),
InstalledLabDockerMount(
_translate_work_path(
plan_path,
plan_path.parent,
controller_root=self.controller_work_root,
engine_root=self.engine_work_root,
),
INSTALLED_LAB_PLAN_PATH,
str(PurePosixPath(INSTALLED_LAB_PLAN_PATH).parent),
True,
),
InstalledLabDockerMount(
@@ -586,8 +588,10 @@ class InstalledLabPackageProfileRunner:
"com.nodedc.definition-sha256": plan.definition_sha256,
"com.nodedc.job-id": plan.job_id,
"com.nodedc.managed-by": "mission-core-worker",
"com.nodedc.module-id": container.container_id,
"com.nodedc.package-sha256": self.package.package_sha256,
"com.nodedc.product": "mission-core",
"com.nodedc.role": "ai-module",
"com.nodedc.stack": "observatory",
},
name_token=name_token,
@@ -33,7 +33,7 @@ INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA: Final = (
INSTALLED_LAB_CONTAINER_IO_SCHEMA: Final = "missioncore.observatory-installed-lab-container-io/v2"
INSTALLED_LAB_SOURCE_ROOT: Final = "/missioncore/input/source"
INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/input/run-plan.json"
INSTALLED_LAB_PLAN_PATH: Final = "/missioncore/plan/run-plan.json"
INSTALLED_LAB_STEP_INPUT_ROOT: Final = "/missioncore/input/steps"
INSTALLED_LAB_RESULT_ROOT: Final = "/missioncore/output"
INSTALLED_LAB_WORK_ROOT: Final = "/missioncore/work"
+183
View File
@@ -0,0 +1,183 @@
"""Durable operator display profiles for Observatory LAB results."""
from __future__ import annotations
import hashlib
import json
import math
import os
import re
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Final, cast
PROFILE_SCHEMA: Final = "missioncore.observatory-lab-view-profile/v1"
_RESULT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,191}\Z")
_COLOR_MODES: Final = {"intensity", "height", "distance", "rgb", "class"}
_PALETTES: Final = {"turbo", "viridis", "plasma", "grayscale"}
class LabViewProfileError(ValueError):
"""A LAB display profile is invalid or unavailable."""
@dataclass(frozen=True, slots=True)
class LabSceneProfile:
point_size: float
accumulation_seconds: float
color_mode: str
palette: str
show_grid: bool
show_labels: bool
show_camera_frustums: bool
def __post_init__(self) -> None:
if (
isinstance(self.point_size, bool)
or not math.isfinite(self.point_size)
or self.point_size < 0.1
or isinstance(self.accumulation_seconds, bool)
or not math.isfinite(self.accumulation_seconds)
or self.accumulation_seconds < 0
or self.color_mode not in _COLOR_MODES
or self.palette not in _PALETTES
or any(
not isinstance(value, bool)
for value in (
self.show_grid,
self.show_labels,
self.show_camera_frustums,
)
)
):
raise LabViewProfileError("invalid LAB scene profile")
def as_dict(self) -> dict[str, object]:
return {
"point_size": self.point_size,
"accumulation_seconds": self.accumulation_seconds,
"color_mode": self.color_mode,
"palette": self.palette,
"show_grid": self.show_grid,
"show_labels": self.show_labels,
"show_camera_frustums": self.show_camera_frustums,
}
@dataclass(frozen=True, slots=True)
class LabViewProfile:
result_id: str
scene_settings: LabSceneProfile
updated_at_utc: str
def __post_init__(self) -> None:
if _RESULT_ID.fullmatch(self.result_id) is None or not self.updated_at_utc.endswith("Z"):
raise LabViewProfileError("invalid LAB view profile identity")
def as_dict(self) -> dict[str, object]:
return {
"schema_version": PROFILE_SCHEMA,
"result_id": self.result_id,
"scene_settings": self.scene_settings.as_dict(),
"updated_at_utc": self.updated_at_utc,
}
class LabViewProfileStore:
"""Mutable, atomic display state keyed by immutable LAB result identity."""
def __init__(self, root: Path) -> None:
root = root.expanduser().absolute()
root.mkdir(parents=True, exist_ok=True, mode=0o700)
if root.is_symlink() or root.resolve() != root:
raise LabViewProfileError("LAB view profile store must be a real directory")
self.root = root
def _path(self, result_id: str) -> Path:
if _RESULT_ID.fullmatch(result_id) is None:
raise LabViewProfileError("invalid LAB result identity")
digest = hashlib.sha256(result_id.encode("utf-8")).hexdigest()
return self.root / f"{digest}.json"
def get(self, result_id: str) -> LabViewProfile | None:
path = self._path(result_id)
if not path.exists():
return None
if path.is_symlink() or not path.is_file() or path.stat().st_size > 64 * 1024:
raise LabViewProfileError("LAB view profile is unavailable")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise LabViewProfileError("LAB view profile is unreadable") from exc
return _decode(value, expected_result_id=result_id)
def save(self, profile: LabViewProfile) -> LabViewProfile:
destination = self._path(profile.result_id)
payload = (
json.dumps(
profile.as_dict(),
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
+ b"\n"
)
descriptor, temporary = tempfile.mkstemp(prefix=".view-profile-", dir=self.root)
path = Path(temporary)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
path.chmod(0o600)
path.replace(destination)
return profile
finally:
path.unlink(missing_ok=True)
def _decode(value: object, *, expected_result_id: str) -> LabViewProfile:
keys = {"schema_version", "result_id", "scene_settings", "updated_at_utc"}
if (
not isinstance(value, dict)
or set(value) != keys
or value.get("schema_version") != PROFILE_SCHEMA
):
raise LabViewProfileError("invalid LAB view profile document")
row = cast(dict[str, object], value)
if row["result_id"] != expected_result_id or not isinstance(row["updated_at_utc"], str):
raise LabViewProfileError("LAB view profile identity changed")
settings = row["scene_settings"]
setting_keys = {
"point_size",
"accumulation_seconds",
"color_mode",
"palette",
"show_grid",
"show_labels",
"show_camera_frustums",
}
if not isinstance(settings, dict) or set(settings) != setting_keys:
raise LabViewProfileError("invalid LAB scene profile document")
scene = cast(dict[str, object], settings)
if (
not isinstance(scene["point_size"], (int, float))
or not isinstance(scene["accumulation_seconds"], (int, float))
or not isinstance(scene["color_mode"], str)
or not isinstance(scene["palette"], str)
):
raise LabViewProfileError("invalid LAB scene profile values")
return LabViewProfile(
result_id=expected_result_id,
scene_settings=LabSceneProfile(
point_size=float(scene["point_size"]),
accumulation_seconds=float(scene["accumulation_seconds"]),
color_mode=scene["color_mode"],
palette=scene["palette"],
show_grid=scene["show_grid"], # type: ignore[arg-type]
show_labels=scene["show_labels"], # type: ignore[arg-type]
show_camera_frustums=scene["show_camera_frustums"], # type: ignore[arg-type]
),
updated_at_utc=row["updated_at_utc"],
)
@@ -32,7 +32,8 @@ from typing import TYPE_CHECKING, Final, cast
import numpy as np
import numpy.typing as npt
from k1link.compute.lidar_replay import LidarReplayPackV2, build_lidar_replay_pack_v2
from k1link.compute.lidar_preparation import prepare_lidar_replay_pack_v2
from k1link.compute.lidar_replay import LidarReplayPackV2
from k1link.observatory.portable_result_contract import canonical_json
from k1link.observatory.recorded_progress import report_recorded_progress
from k1link.observatory.source_admission import (
@@ -225,7 +226,7 @@ def materialize_m49_portable_source_from_worker_stage(
if parent.is_symlink() or not parent.is_dir():
raise M49PortableSourceError("portable M4.9 output parent is unsafe")
try:
lidar_pack_root = build_lidar_replay_pack_v2(
lidar_pack_root = prepare_lidar_replay_pack_v2(
root / "mqtt.raw.k1mqtt",
parent / "lidar-replay-packs",
session_id=job.source_session_id,
@@ -538,7 +539,9 @@ def _materialize_stage(
f"\t{anchor.session_seconds:.9f}\t-1\t0"
)
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
"source-preparation",
anchor.timeline_frame_index + 1,
len(anchors),
)
continue
@@ -600,7 +603,9 @@ def _materialize_stage(
)
available_slot += 1
report_recorded_progress(
"source-preparation", anchor.timeline_frame_index + 1, len(anchors),
"source-preparation",
anchor.timeline_frame_index + 1,
len(anchors),
)
if available_slot < 1:
@@ -0,0 +1,457 @@
"""Immutable, data-only AI compositions shared by Core and the installed Worker.
Selections contain no executable instructions. Installed module identities own
their containers; the graph connects typed capabilities, never historical LABs.
"""
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Final, cast
from k1link.observatory.portable_run_definitions import canonical_sha256
COMPOSITION_SCHEMA: Final = "missioncore.observatory-ai-composition/v1"
MODULE_SCHEMA: Final = "missioncore.observatory-ai-module/v1"
NODE_INPUT_SCHEMA: Final = "missioncore.observatory-ai-node-input/v1"
GROUPS: Final = ("segmentation", "detection", "geometry", "range", "motion", "policy")
_ID = re.compile(r"[a-z][a-z0-9.-]{1,95}\Z")
_SHA = re.compile(r"[a-f0-9]{64}\Z")
class CompositionError(ValueError):
"""A selection, dependency or immutable identity is not admitted."""
def canonical_bytes(value: object) -> bytes:
try:
return json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
).encode("utf-8")
except (TypeError, ValueError) as exc:
raise CompositionError("composition must contain finite JSON data") from exc
def require_digest(value: object) -> str:
if not isinstance(value, str) or not _SHA.fullmatch(value):
raise CompositionError("invalid immutable digest")
return value
def _identifier(value: object) -> str:
if not isinstance(value, str) or not _ID.fullmatch(value):
raise CompositionError("invalid module or capability identifier")
return value
def _object(value: object, keys: set[str]) -> dict[str, object]:
if not isinstance(value, dict) or set(value) != keys:
raise CompositionError("unexpected composition fields")
return cast(dict[str, object], value)
@dataclass(frozen=True, slots=True)
class ModuleSpec:
"""An installed version; all semantic and execution identities are sealed.
Source ports use the source.* namespace. Preparation modules are installed
infrastructure and are added only when a selected module consumes a port.
"""
module_id: str
label: str
group: str
image_sha256: str
implementation_sha256: str
model_sha256: str | None
contract_sha256: str
requires: tuple[str, ...]
provides: tuple[str, ...]
optional_inputs: tuple[str, ...] = ()
parameter_choices_json: bytes = b"{}"
defaults_json: bytes = b"{}"
state_policy: str = "stateless"
def __post_init__(self) -> None:
_identifier(self.module_id)
if self.group not in (*GROUPS, "preparation"):
raise CompositionError("unknown functional group")
if not self.label or len(self.label) > 120:
raise CompositionError("invalid module label")
for digest in (self.image_sha256, self.implementation_sha256, self.contract_sha256):
require_digest(digest)
if self.model_sha256 is not None:
require_digest(self.model_sha256)
for ports in (self.requires, self.provides, self.optional_inputs):
if tuple(sorted(set(ports))) != ports:
raise CompositionError("module ports must be unique and sorted")
for port in ports:
_identifier(port)
if not self.provides or any(port.startswith("source.") for port in self.provides):
raise CompositionError("a module cannot provide raw source authority")
if set(self.requires) & set(self.optional_inputs):
raise CompositionError("required and optional ports overlap")
if self.state_policy not in ("stateless", "causal-reset-at-source-start"):
raise CompositionError("unqualified temporal state policy")
choices = json.loads(self.parameter_choices_json)
defaults = json.loads(self.defaults_json)
if not isinstance(choices, dict) or not isinstance(defaults, dict):
raise CompositionError("module parameters must be objects")
if set(defaults) != set(choices):
raise CompositionError("every parameter needs an explicit default")
for key, values in choices.items():
_identifier(key)
if not isinstance(values, list) or not values:
raise CompositionError("parameter choices must be finite nonempty lists")
self.resolve_parameters(defaults)
def resolve_parameters(self, supplied: object) -> dict[str, object]:
choices = cast(dict[str, list[object]], json.loads(self.parameter_choices_json))
if not isinstance(supplied, dict) or set(supplied) - set(choices):
raise CompositionError("parameter is not installed for this module")
resolved: dict[str, object] = {
**cast(dict[str, object], json.loads(self.defaults_json)),
**supplied,
}
for key, value in resolved.items():
# JSON identities distinguish true from 1 and reject nonfinite floats.
if canonical_bytes(value) not in [canonical_bytes(item) for item in choices[key]]:
raise CompositionError(f"unsupported value for {key}")
return resolved
def identity_document(self) -> dict[str, object]:
return {
"schema_version": MODULE_SCHEMA,
"module_id": self.module_id,
"group": self.group,
"image_sha256": self.image_sha256,
"implementation_sha256": self.implementation_sha256,
"model_sha256": self.model_sha256,
"contract_sha256": self.contract_sha256,
"requires": list(self.requires),
"provides": list(self.provides),
"optional_inputs": list(self.optional_inputs),
"parameter_choices": json.loads(self.parameter_choices_json),
"defaults": json.loads(self.defaults_json),
"state_policy": self.state_policy,
}
@property
def sha256(self) -> str:
return canonical_sha256(self.identity_document())
@property
def docker_name(self) -> str:
return f"ndc-mission-core-ai-module-{self.module_id}"
def docker_labels(self) -> dict[str, str]:
return {
"com.nodedc.product": "mission-core",
"com.nodedc.stack": "observatory",
"com.nodedc.role": "ai-module",
"com.nodedc.managed-by": "mission-core-worker",
"com.nodedc.module-id": self.module_id,
"com.nodedc.module-sha256": self.sha256,
}
@dataclass(frozen=True, slots=True)
class CompositionNode:
module: ModuleSpec
parameters_json: bytes
# Capability -> provider module ID, or source.* for authoritative source.
inputs: tuple[tuple[str, str], ...]
def as_dict(self) -> dict[str, object]:
return {
"module_id": self.module.module_id,
"module_sha256": self.module.sha256,
"parameters": json.loads(self.parameters_json),
"inputs": dict(self.inputs),
}
@dataclass(frozen=True, slots=True)
class CompositionSpec:
"""Source-independent graph, in deterministic topological execution order."""
nodes: tuple[CompositionNode, ...]
@property
def source_capabilities(self) -> tuple[str, ...]:
return tuple(
sorted(
{
provider
for node in self.nodes
for _, provider in node.inputs
if provider.startswith("source.")
}
)
)
@property
def outputs(self) -> tuple[str, ...]:
return tuple(
sorted(
{
port
for node in self.nodes
if node.module.group != "preparation"
for port in node.module.provides
}
)
)
def as_dict(self) -> dict[str, object]:
return {
"schema_version": COMPOSITION_SCHEMA,
"nodes": [node.as_dict() for node in self.nodes],
"source_capabilities": list(self.source_capabilities),
"outputs": list(self.outputs),
"execution": {"max_parallel_nodes": 1, "mode": "recorded-observation-only"},
}
@property
def sha256(self) -> str:
return canonical_sha256(self.as_dict())
def selection_document(self) -> dict[str, object]:
return {
"schema_version": COMPOSITION_SCHEMA,
"selections": [
{
"group": node.module.group,
"module_id": node.module.module_id,
"module_sha256": node.module.sha256,
"parameters": json.loads(node.parameters_json),
}
for node in self.nodes
if node.module.group != "preparation"
],
}
class ModuleRegistry:
def __init__(self, modules: tuple[ModuleSpec, ...]) -> None:
self.modules = modules
self._by_id = {module.module_id: module for module in modules}
if len(self._by_id) != len(modules):
raise CompositionError("duplicate installed module ID")
@classmethod
def from_file(cls, path: Path) -> ModuleRegistry:
candidate = path.expanduser().absolute()
if (
candidate.is_symlink()
or not candidate.is_file()
or candidate.stat().st_size > 1024 * 1024
):
raise CompositionError("AI module registry must be a bounded regular file")
try:
root = _object(json.loads(candidate.read_text()), {"schema_version", "modules"})
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise CompositionError("AI module registry is unreadable") from exc
if root["schema_version"] != "missioncore.observatory-ai-module-registry/v1":
raise CompositionError("unsupported AI module registry schema")
if not isinstance(root["modules"], list):
raise CompositionError("AI module registry rows must be an array")
return cls(tuple(_module_from_dict(value) for value in root["modules"]))
def catalog(self) -> dict[str, object]:
return {
"schema_version": "missioncore.observatory-ai-module-catalog/v1",
"groups": [
{
"group": group,
"modules": [
{
"module_id": module.module_id,
"module_sha256": module.sha256,
"label": module.label,
"docker_name": module.docker_name,
"requires": list(module.requires),
"provides": list(module.provides),
"parameter_choices": json.loads(module.parameter_choices_json),
"defaults": json.loads(module.defaults_json),
}
for module in self.modules
if module.group == group
],
}
for group in GROUPS
],
}
def compose(self, document: object) -> CompositionSpec:
root = _object(document, {"schema_version", "selections"})
if root["schema_version"] != COMPOSITION_SCHEMA:
raise CompositionError("unsupported composition schema")
selections = root["selections"]
if not isinstance(selections, list) or not 1 <= len(selections) <= len(GROUPS):
raise CompositionError("select at least one AI module")
selected: dict[str, tuple[ModuleSpec, bytes]] = {}
groups: set[str] = set()
for raw in selections:
row = _object(raw, {"group", "module_id", "module_sha256", "parameters"})
module = self._by_id.get(_identifier(row["module_id"]))
if module is None or module.sha256 != row["module_sha256"]:
raise CompositionError("module version is not installed")
if module.group != row["group"] or module.group not in GROUPS:
raise CompositionError("module belongs to another functional group")
if module.group in groups:
raise CompositionError(f"select only one provider for {module.group}")
groups.add(module.group)
selected[module.module_id] = (
module,
canonical_bytes(module.resolve_parameters(row["parameters"])),
)
def producers() -> dict[str, str]:
result: dict[str, str] = {}
for module, _ in selected.values():
for port in module.provides:
if port in result:
raise CompositionError(f"ambiguous provider for {port}")
result[port] = module.module_id
return result
# Only source preparation is implicit. Missing analytical dependencies
# must be selected by the operator, never silently added to the LAB.
while True:
supplied = producers()
missing = sorted(
{
port
for module, _ in selected.values()
for port in module.requires
if not port.startswith("source.") and port not in supplied
}
)
if not missing:
break
added = False
for port in missing:
candidates = [
module
for module in self.modules
if module.group == "preparation" and port in module.provides
]
if len(candidates) != 1:
raise CompositionError(f"select a module providing {port}")
module = candidates[0]
if module.module_id not in selected:
selected[module.module_id] = (
module,
canonical_bytes(module.resolve_parameters({})),
)
added = True
if not added:
raise CompositionError("unresolved module dependencies")
supplied = producers()
pending: dict[str, CompositionNode] = {}
for module, parameters in selected.values():
ports = (
*module.requires,
*(port for port in module.optional_inputs if port in supplied),
)
inputs = tuple(
sorted(
(port, port if port.startswith("source.") else supplied[port]) for port in ports
)
)
pending[module.module_id] = CompositionNode(module, parameters, inputs)
ordered: list[CompositionNode] = []
emitted: set[str] = set()
while pending:
ready = sorted(
key
for key, node in pending.items()
if all(
provider.startswith("source.") or provider in emitted
for _, provider in node.inputs
)
)
if not ready:
raise CompositionError("cyclic module dependencies")
for key in ready:
ordered.append(pending.pop(key))
emitted.add(key)
return CompositionSpec(tuple(ordered))
def node_input_identity(
node: CompositionNode,
inputs: dict[str, str],
*,
state_context_sha256: str | None = None,
) -> dict[str, object]:
"""Exact node cache identity, independent of job and unrelated selections.
Each input digest seals bytes AND its I/O envelope (clock, calibration,
preprocessing, cadence, precision, unavailable samples). Stateful nodes
additionally bind their initialization and causal history.
"""
if set(inputs) != {port for port, _ in node.inputs}:
raise CompositionError("node input capabilities disagree with the graph")
for digest in inputs.values():
require_digest(digest)
if node.module.state_policy == "stateless":
if state_context_sha256 is not None:
raise CompositionError("stateless node cannot bind temporal state")
else:
require_digest(state_context_sha256)
return {
"schema_version": NODE_INPUT_SCHEMA,
"module_sha256": node.module.sha256,
"parameters": json.loads(node.parameters_json),
"inputs": dict(sorted(inputs.items())),
"state_policy": node.module.state_policy,
"state_context_sha256": state_context_sha256,
}
def _module_from_dict(value: object) -> ModuleSpec:
row = _object(
value,
{
"module_id",
"label",
"group",
"image_sha256",
"implementation_sha256",
"model_sha256",
"contract_sha256",
"requires",
"provides",
"optional_inputs",
"parameter_choices",
"defaults",
"state_policy",
},
)
arrays: dict[str, tuple[str, ...]] = {}
for name in ("requires", "provides", "optional_inputs"):
raw = row[name]
if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw):
raise CompositionError(f"module {name} must be an array of strings")
arrays[name] = tuple(raw)
return ModuleSpec(
module_id=_identifier(row["module_id"]),
label=row["label"] if isinstance(row["label"], str) else "",
group=row["group"] if isinstance(row["group"], str) else "",
image_sha256=require_digest(row["image_sha256"]),
implementation_sha256=require_digest(row["implementation_sha256"]),
model_sha256=None if row["model_sha256"] is None else require_digest(row["model_sha256"]),
contract_sha256=require_digest(row["contract_sha256"]),
requires=arrays["requires"],
provides=arrays["provides"],
optional_inputs=arrays["optional_inputs"],
parameter_choices_json=canonical_bytes(row["parameter_choices"]),
defaults_json=canonical_bytes(row["defaults"]),
state_policy=row["state_policy"] if isinstance(row["state_policy"], str) else "",
)
@@ -0,0 +1,49 @@
"""Durable immutable composition catalog owned by Core."""
from __future__ import annotations
import os
import tempfile
from pathlib import Path
from k1link.observatory.modular_composition import (
CompositionError,
CompositionSpec,
ModuleRegistry,
canonical_bytes,
)
class ModularCompositionStore:
def __init__(self, root: Path, registry: ModuleRegistry) -> None:
root = root.expanduser().absolute()
root.mkdir(parents=True, exist_ok=True, mode=0o700)
if root.is_symlink() or root.resolve() != root:
raise CompositionError("composition store must be a real directory")
self.root = root
self.registry = registry
def save(self, selection: object) -> tuple[CompositionSpec, bool]:
composition = self.registry.compose(selection)
destination = self.root / f"{composition.sha256}.json"
payload = canonical_bytes(composition.as_dict())
if destination.exists():
if destination.is_symlink() or destination.read_bytes() != payload:
raise CompositionError("stored composition identity is damaged")
return composition, False
descriptor, temporary = tempfile.mkstemp(prefix=".composition-", dir=self.root)
path = Path(temporary)
try:
with os.fdopen(descriptor, "wb") as stream:
stream.write(payload)
stream.flush()
os.fsync(stream.fileno())
path.chmod(0o444)
try:
path.rename(destination)
except OSError:
if destination.is_symlink() or destination.read_bytes() != payload:
raise CompositionError("composition publication conflict") from None
return composition, True
finally:
path.unlink(missing_ok=True)
@@ -0,0 +1,528 @@
"""Installed-package prepare and result steps for one independent AI module."""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import sys
from collections.abc import Mapping, Sequence
from datetime import UTC, datetime
from pathlib import Path
from typing import Final, cast
from k1link.observatory.installed_lab_packages import (
INSTALLED_LAB_PLAN_PATH,
INSTALLED_LAB_RESULT_ROOT,
INSTALLED_LAB_SOURCE_ROOT,
INSTALLED_LAB_STEP_INPUT_ROOT,
)
from k1link.observatory.m49_portable_source import (
M49_PORTABLE_STAGE_MANIFEST,
materialize_m49_portable_source_from_worker_stage,
)
from k1link.observatory.modular_result import MODULAR_RESULT_KIND, MODULAR_RESULT_SCHEMA
from k1link.observatory.portable_lab_v1_executor import (
build_portable_ddrnet_effective_config,
portable_lab_v1_source_input_from_document,
)
from k1link.observatory.portable_lab_v1_local_runners import (
PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
)
from k1link.observatory.portable_lab_v1_worker import (
_SealedPackageJobView,
materialize_recorded_camera_source_from_worker_stage,
)
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
RESULT_DOCUMENT_ROLE,
RESULT_PACKAGE_MANIFEST_NAME,
PortableResultArtifact,
PortableResultPackageManifest,
canonical_json,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
PortableRunDefinitionRegistry,
canonical_sha256,
)
from k1link.observatory.portable_worker_runtime import PortableWorkerSourceStage
from k1link.observatory.recorded_jobs import RecordedExecutorIdentity
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
MODULAR_PACKAGE_CONTRACT_SCHEMA: Final = (
"missioncore.observatory-ai-module-installed-package-contract/v1"
)
_CONTRACT = Path("/opt/nodedc/package/contract.json")
_DEFINITIONS = Path("/opt/nodedc/package/portable-run-definitions.json")
_DDRNET_PROFILE = Path("/opt/nodedc/package/ddrnet-profile.json")
_M49_PROFILE = Path("/opt/nodedc/package/m49-profile.json")
_PREPARE = Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "prepare"
_MAX_DOCUMENT_BYTES: Final = 2 * 1024 * 1024
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
_ARTIFACTS: Final = {
"ddrnet": (
("ddrnet-decode-repair", "decode-repair.json", "application/json"),
("ddrnet-result-document", "result.json", "application/json"),
("ddrnet-semantic-mask-archive", "semantic-masks.zip", "application/zip"),
),
"eomt": (
("eomt-decode-repair", "decode-repair.json", "application/json"),
("eomt-panoptic-frame-metadata", "frames.jsonl", "application/x-ndjson"),
("eomt-gpu-telemetry", "gpu-telemetry.jsonl", "application/x-ndjson"),
("eomt-panoptic-mask-archive", "masks.tar.gz", "application/gzip"),
("eomt-overlay-video", "perception.mp4", "video/mp4"),
("eomt-result-document", "result.json", "application/json"),
("eomt-run-report", "run-report.json", "application/json"),
("eomt-source-frame-manifest", "source-frames.json", "application/json"),
),
"rf-detr": (
("rf-detr-frame-detections", "detections.jsonl", "application/x-ndjson"),
("rf-detr-result-document", "result.json", "application/json"),
),
"object-distance": (
("rf-detr-frame-detections", "detections.jsonl", "application/x-ndjson"),
("rf-detr-result-document", "result.json", "application/json"),
(
"object-distance-frame-observations",
"object-distances.jsonl",
"application/x-ndjson",
),
("object-distance-result-document", "result.json", "application/json"),
),
}
class ModularPackageStepError(RuntimeError):
"""The selected installed module package changed or is incomplete."""
def main(argv: Sequence[str] | None = None) -> int:
arguments = tuple(sys.argv[1:] if argv is None else argv)
if arguments == ("prepare",):
prepare()
return 0
if arguments == ("assemble",):
assemble()
return 0
raise ModularPackageStepError("AI-module package step is not allowlisted")
def prepare() -> None:
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "AI-module prepare output")
contract = _load_contract()
runtime_plan = _runtime_plan()
definition = _definition(runtime_plan)
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
module = _object(contract["module"], "AI module")
module_id = _identifier(module["module_id"], "module id")
stage = PortableWorkerSourceStage(
root=_real_directory(Path(INSTALLED_LAB_SOURCE_ROOT), "AI-module source"),
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "source bundle"),
source_capability_manifest_sha256=_digest(
runtime_plan["source_capability_manifest_sha256"], "source capability"
),
source_adapter_sha256=_digest(runtime_plan["source_adapter_sha256"], "source adapter"),
)
materialized = materialize_recorded_camera_source_from_worker_stage(
worker_stage=stage,
job=job,
definition=definition,
output_parent=output,
)
camera_target = output / "camera-job"
camera_stage_parent = materialized.camera_job_root.parent
os.replace(materialized.camera_job_root, camera_target)
camera_stage_parent.rmdir()
materialized.root.rmdir()
source = materialized.descriptor.as_dict()
_write(output / "source-input.json", source)
if module_id == "object-distance":
spatial_root = output / "spatial-source"
m49 = materialize_m49_portable_source_from_worker_stage(
worker_stage=stage,
job=job,
profile_path=_M49_PROFILE,
output_parent=spatial_root,
)
manifest = _load_object(
m49.root / M49_PORTABLE_STAGE_MANIFEST,
"object-distance spatial source",
)
identity = _object(manifest.get("identity"), "object-distance spatial identity")
lidar = _object(identity.get("lidar_replay"), "object-distance LiDAR identity")
pack_id = _identifier(lidar.get("pack_id"), "object-distance LiDAR pack id")
pack = _real_directory(
spatial_root / "lidar-replay-packs" / pack_id,
"object-distance LiDAR pack",
)
os.replace(pack, output / "lidar-pack")
os.replace(m49.root, output / "m49-source")
shutil.rmtree(spatial_root)
plan_sha = canonical_sha256(
{
"schema_version": "missioncore.observatory-ai-module-plan/v1",
"job_identity_sha256": job.identity_sha256,
"definition_sha256": definition.definition_sha256,
"module_id": module_id,
"module_sha256": _digest(module["module_sha256"], "module identity"),
"source_input_sha256": materialized.descriptor.identity_sha256,
}
)
assets = _object(contract["component_assets"], "component assets")
images = _object(contract["component_images"], "component images")
_write(
output / "camera-source-request.json",
_component_request(
component="camera-source",
image_sha=_digest(images["camera-source"], "camera image"),
source=source,
plan_sha=plan_sha,
definition=definition,
release_sha=_digest(runtime_plan["candidate_sha256"], "runtime candidate"),
assets=_asset_rows(assets["camera-source"], "camera-source"),
ddrnet_config_sha=None,
),
)
ddrnet_config_sha: str | None = None
if module_id == "ddrnet":
profile = _load_object(_DDRNET_PROFILE, "DDRNet profile")
effective = build_portable_ddrnet_effective_config(
profile,
source=materialized.descriptor,
)
_write(output / "effective-ddrnet-config.json", effective)
ddrnet_config_sha = canonical_sha256(effective)
_write(
output / f"{module_id}-request.json",
_component_request(
component=module_id,
image_sha=_digest(images[module_id], "module image"),
source=source,
plan_sha=plan_sha,
definition=definition,
release_sha=_digest(runtime_plan["candidate_sha256"], "runtime candidate"),
assets=_asset_rows(assets[module_id], module_id),
ddrnet_config_sha=ddrnet_config_sha,
),
)
def assemble() -> None:
output = _empty_directory(Path(INSTALLED_LAB_RESULT_ROOT), "AI-module result output")
artifacts_root = output / "artifacts"
artifacts_root.mkdir(mode=0o700)
contract = _load_contract()
runtime_plan = _runtime_plan()
definition = _definition(runtime_plan)
job = _sealed_job(runtime_plan, definition=definition, contract=contract)
source_input = portable_lab_v1_source_input_from_document(
_load_object(_PREPARE / "source-input.json", "AI-module source input")
)
module = _object(contract["module"], "AI module")
module_id = _identifier(module["module_id"], "module id")
module_root = _real_directory(
Path(INSTALLED_LAB_STEP_INPUT_ROOT) / module_id, "AI-module output"
)
component_result = _regular_file(module_root / "result.json", module_root, "module result")
component_result_sha = _sha256(component_result)
artifacts: list[PortableResultArtifact] = []
for role, name, media_type in _ARTIFACTS[module_id]:
artifact_root = (
_real_directory(
Path(INSTALLED_LAB_STEP_INPUT_ROOT) / "rf-detr",
"RF-DETR dependency output",
)
if module_id == "object-distance" and role.startswith("rf-detr-")
else module_root
)
source_path = _regular_file(artifact_root / name, artifact_root, role)
artifact_name = f"{role}{source_path.suffix}"
if name.endswith(".tar.gz"):
artifact_name = f"{role}.tar.gz"
target_name = f"artifacts/{artifact_name}"
target = artifacts_root / artifact_name
_copy(source_path, target)
artifacts.append(_artifact(role, target_name, media_type, target))
source = {
"session_id": job.source_session_id,
"catalog_sha256": job.source_catalog_sha256,
"bundle_sha256": job.source_bundle_sha256,
"capability_manifest_sha256": job.source_capability_manifest_sha256,
"camera_input_sha256": source_input.camera_input_sha256,
"frame_count": source_input.frame_count,
"timeline_start_seconds": source_input.timeline_start_seconds,
"timeline_end_seconds": source_input.timeline_end_seconds,
}
module_view = {
"module_id": module_id,
"label": _text(module["label"], "module label"),
"module_sha256": _digest(module["module_sha256"], "module identity"),
"image_sha256": _digest(module["image_sha256"], "module image"),
"definition_sha256": definition.definition_sha256,
"component_result_sha256": component_result_sha,
}
identity = {
"job_identity_sha256": job.identity_sha256,
"source_bundle_sha256": job.source_bundle_sha256,
"definition_sha256": definition.definition_sha256,
"module_id": module_id,
"component_result_sha256": component_result_sha,
}
identity_sha = canonical_sha256(identity)
result_id = f"ai-layer-{module_id}-{identity_sha}"
result_document = {
"schema_version": MODULAR_RESULT_SCHEMA,
"result_id": result_id,
"result_kind": MODULAR_RESULT_KIND,
"identity": identity,
"identity_sha256": identity_sha,
"source": source,
"module": module_view,
"artifacts": [item.as_dict() for item in sorted(artifacts, key=lambda item: item.role)],
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
}
result_path = artifacts_root / "result.json"
_write(result_path, result_document)
artifacts.append(
_artifact(
RESULT_DOCUMENT_ROLE,
"artifacts/result.json",
"application/json",
result_path,
)
)
manifest = PortableResultPackageManifest.create(
job=_SealedPackageJobView.from_job(job),
definition=definition,
result_id=result_id,
created_at_utc=datetime.now(UTC).isoformat().replace("+00:00", "Z"),
artifacts=artifacts,
)
(output / RESULT_PACKAGE_MANIFEST_NAME).write_bytes(manifest.canonical_bytes)
os.chmod(output / RESULT_PACKAGE_MANIFEST_NAME, 0o400)
def _component_request(
*,
component: str,
image_sha: str,
source: Mapping[str, object],
plan_sha: str,
definition: PortableRunDefinition,
release_sha: str,
assets: list[dict[str, object]],
ddrnet_config_sha: str | None,
) -> dict[str, object]:
prepared = f"{INSTALLED_LAB_STEP_INPUT_ROOT}/prepare"
return {
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
"component": component,
"component_image_sha256": image_sha,
"plan_sha256": plan_sha,
"definition_sha256": definition.definition_sha256,
"release_candidate_sha256": release_sha,
"source": dict(source),
"paths": {
"camera_job_root": f"{prepared}/camera-job",
"request": f"{prepared}/{component}-request.json",
"output_root": INSTALLED_LAB_RESULT_ROOT,
"effective_ddrnet_config": (
f"{prepared}/effective-ddrnet-config.json" if component == "ddrnet" else None
),
"eomt_result_root": (
f"{INSTALLED_LAB_STEP_INPUT_ROOT}/camera-source" if component == "ddrnet" else None
),
"decoded_frames_root": (
f"{INSTALLED_LAB_RESULT_ROOT}/source-frames"
if component == "camera-source"
else f"{INSTALLED_LAB_STEP_INPUT_ROOT}/camera-source/source-frames"
),
},
"effective_ddrnet_config_sha256": ddrnet_config_sha,
"assets": assets,
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
}
def _sealed_job(
runtime_plan: Mapping[str, object],
*,
definition: PortableRunDefinition,
contract: Mapping[str, object],
) -> SealedObservatoryRecordedJob:
executor = _object(contract["executor"], "package executor")
identity = RecordedExecutorIdentity(
release_sha256=_digest(executor["release_sha256"], "executor release"),
image_sha256=_digest(executor["image_sha256"], "executor image"),
model_manifest_sha256=definition.model_manifest_sha256,
resource_profile_sha256=definition.resource_profile.profile_sha256,
)
return SealedObservatoryRecordedJob(
job_id=_text(runtime_plan["job_id"], "job id"),
request_sha256=_digest(runtime_plan["request_sha256"], "request"),
identity_sha256=_digest(runtime_plan["identity_sha256"], "job identity"),
submission_receipt_sha256=_digest(runtime_plan["submission_receipt_sha256"], "receipt"),
source_session_id=_text(runtime_plan["source_session_id"], "source session"),
source_catalog_sha256=_digest(runtime_plan["source_catalog_sha256"], "catalog"),
source_bundle_sha256=_digest(runtime_plan["source_bundle_sha256"], "bundle"),
source_capability_manifest_sha256=_digest(
runtime_plan["source_capability_manifest_sha256"], "capability"
),
source_adapter_id=_identifier(runtime_plan["source_adapter_id"], "adapter id"),
source_adapter_version=_positive_int(
runtime_plan["source_adapter_version"], "adapter version"
),
source_adapter_sha256=_digest(runtime_plan["source_adapter_sha256"], "adapter"),
setup_id=definition.setup_id,
definition_id=definition.definition_id,
definition_version=definition.version,
definition_sha256=definition.definition_sha256,
executor_release_id=_identifier(executor["release_id"], "executor release id"),
executor_identity=identity,
model_release_ids=definition.learned_models,
resource_profile_id=definition.resource_profile.profile_id,
checkpoint_policy=definition.resource_profile.checkpoint_policy,
allowed_checkpoints=definition.resource_profile.allowed_checkpoints,
claim_generation=_positive_int(runtime_plan["claim_generation"], "claim generation"),
claim_claimed_at_utc=None,
claim_expires_at_utc=None,
claim_heartbeat_at_utc=None,
claim_renewal_count=0,
restart_from_zero=False,
)
def _runtime_plan() -> dict[str, object]:
outer = _load_object(Path(INSTALLED_LAB_PLAN_PATH), "installed AI-module run plan")
if (
outer.get("schema_version") != "missioncore.observatory-installed-lab-run-plan/v1"
or outer.get("authority") != OBSERVATION_ONLY_AUTHORITY
):
raise ModularPackageStepError("installed AI-module run plan changed")
return _object(outer.get("runtime_plan"), "portable runtime plan")
def _definition(runtime_plan: Mapping[str, object]) -> PortableRunDefinition:
return PortableRunDefinitionRegistry.from_file(_DEFINITIONS).resolve(
_identifier(runtime_plan["setup_id"], "setup id"),
_digest(runtime_plan["definition_sha256"], "definition"),
)
def _load_contract() -> dict[str, object]:
value = _load_object(_CONTRACT, "AI-module package contract")
if (
value.get("schema_version") != MODULAR_PACKAGE_CONTRACT_SCHEMA
or value.get("authority") != OBSERVATION_ONLY_AUTHORITY
):
raise ModularPackageStepError("AI-module package contract changed")
return value
def _asset_rows(value: object, component: str) -> list[dict[str, object]]:
if not isinstance(value, list):
raise ModularPackageStepError(f"{component} assets are not an array")
rows = [dict(_object(row, f"{component} asset")) for row in value]
ids = tuple(cast(str, row.get("asset_id")) for row in rows)
if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)):
raise ModularPackageStepError(f"{component} assets are not canonical")
return rows
def _artifact(role: str, relative: str, media: str, path: Path) -> PortableResultArtifact:
return PortableResultArtifact(role, relative, media, path.stat().st_size, _sha256(path))
def _copy(source: Path, target: Path) -> None:
with source.open("rb") as reader, target.open("xb") as writer:
shutil.copyfileobj(reader, writer, 1024 * 1024)
os.chmod(target, 0o400)
if target.stat().st_size != source.stat().st_size or _sha256(target) != _sha256(source):
raise ModularPackageStepError("AI-module artifact copy changed")
def _write(path: Path, value: Mapping[str, object]) -> None:
path.write_bytes(canonical_json(value))
os.chmod(path, 0o400)
def _load_object(path: Path, label: str) -> dict[str, object]:
if (
path.is_symlink()
or not path.is_file()
or not 0 < path.stat().st_size <= _MAX_DOCUMENT_BYTES
):
raise ModularPackageStepError(f"{label} is unavailable")
try:
return _object(json.loads(path.read_bytes()), label)
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ModularPackageStepError(f"{label} is invalid JSON") from exc
def _regular_file(path: Path, root: Path, label: str) -> Path:
if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
raise ModularPackageStepError(f"{label} is unavailable")
return path.resolve()
def _empty_directory(path: Path, label: str) -> Path:
root = _real_directory(path, label)
if any(root.iterdir()):
raise ModularPackageStepError(f"{label} is not empty")
return root
def _real_directory(path: Path, label: str) -> Path:
if path.is_symlink() or not path.is_dir():
raise ModularPackageStepError(f"{label} is unavailable")
return path.resolve(strict=True)
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise ModularPackageStepError(f"{label} is invalid")
return cast(dict[str, object], value)
def _text(value: object, label: str) -> str:
if not isinstance(value, str) or not value:
raise ModularPackageStepError(f"{label} is invalid")
return value
def _identifier(value: object, label: str) -> str:
text = _text(value, label)
if _IDENTIFIER.fullmatch(text) is None:
raise ModularPackageStepError(f"{label} is invalid")
return text
def _digest(value: object, label: str) -> str:
text = _text(value, label)
if _SHA256.fullmatch(text) is None:
raise ModularPackageStepError(f"{label} is invalid")
return text
def _positive_int(value: object, label: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise ModularPackageStepError(f"{label} is invalid")
return value
def _sha256(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
if __name__ == "__main__":
try:
raise SystemExit(main())
except ModularPackageStepError as exc:
print(f"installed AI-module package rejected: {exc}", file=sys.stderr)
raise SystemExit(2) from exc
@@ -0,0 +1,196 @@
"""Sealed Worker-local node results; source CAS and final Core LABs are separate.
Entries are published by atomic directory rename only after all regular files
are hashed. Readers validate bytes, not existence. A damaged entry is a miss;
original evidence is never overwritten or removed by a reader.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import stat
import tempfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import cast
from k1link.observatory.modular_composition import (
CompositionError,
canonical_bytes,
require_digest,
)
from k1link.observatory.portable_run_definitions import canonical_sha256
NODE_RESULT_SCHEMA = "missioncore.observatory-ai-node-result/v1"
_MAX_MANIFEST_BYTES = 4 * 1024 * 1024
_MAX_FILES = 20_000
@dataclass(frozen=True, slots=True)
class SealedNodeResult:
root: Path
input_sha256: str
result_sha256: str
manifest_json: bytes
@property
def manifest(self) -> dict[str, object]:
value = json.loads(self.manifest_json)
if not isinstance(value, dict):
raise CompositionError("node manifest must be an object")
return cast(dict[str, object], value)
def _hash_file(path: Path) -> tuple[int, str]:
mode = path.lstat().st_mode
if not stat.S_ISREG(mode):
raise CompositionError("node result must contain only regular files")
digest = hashlib.sha256()
size = 0
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
size += len(chunk)
return size, digest.hexdigest()
def _files(root: Path) -> list[dict[str, object]]:
entries: list[dict[str, object]] = []
for path in sorted(root.rglob("*")):
mode = path.lstat().st_mode
if stat.S_ISDIR(mode):
continue
size, digest = _hash_file(path)
entries.append(
{"path": path.relative_to(root).as_posix(), "byte_length": size, "sha256": digest}
)
if len(entries) > _MAX_FILES:
raise CompositionError("node result exceeds the file limit")
if not entries:
raise CompositionError("empty node output cannot be sealed")
return entries
class ModularNodeCache:
def __init__(self, root: Path) -> None:
root = root.expanduser().absolute()
root.mkdir(parents=True, exist_ok=True, mode=0o700)
if root.is_symlink() or not root.is_dir() or root.resolve() != root:
raise CompositionError("node cache root must be a real directory")
self.root = root
def lookup(self, input_identity: dict[str, object]) -> SealedNodeResult | None:
input_sha256 = canonical_sha256(input_identity)
entry = self.root / input_sha256
try:
if entry.is_symlink() or not entry.is_dir():
return None
manifest_path = entry / "manifest.json"
if (
not stat.S_ISREG(manifest_path.lstat().st_mode)
or manifest_path.stat().st_size > _MAX_MANIFEST_BYTES
):
return None
payload = manifest_path.read_bytes()
manifest = cast(dict[str, object], json.loads(payload))
if set(manifest) != {
"schema_version",
"input_identity",
"input_sha256",
"outputs",
"metadata",
"result_sha256",
}:
return None
identity = {key: value for key, value in manifest.items() if key != "result_sha256"}
if (
manifest["schema_version"] != NODE_RESULT_SCHEMA
or manifest["input_identity"] != input_identity
or manifest["input_sha256"] != input_sha256
or manifest["result_sha256"] != canonical_sha256(identity)
or payload != canonical_bytes(manifest)
):
return None
data = entry / "data"
if data.is_symlink() or not data.is_dir() or _files(data) != manifest["outputs"]:
return None
return SealedNodeResult(data, input_sha256, manifest["result_sha256"], payload)
except (OSError, ValueError, TypeError, KeyError):
return None
def seal(
self,
input_identity: dict[str, object],
output_root: Path,
*,
metadata: dict[str, object],
) -> SealedNodeResult:
existing = self.lookup(input_identity)
if existing is not None:
return existing
input_sha256 = canonical_sha256(input_identity)
require_digest(input_sha256)
destination = self.root / input_sha256
if destination.exists() or destination.is_symlink():
raise CompositionError("damaged node cache entry requires explicit repair")
output_root = output_root.absolute()
if output_root.is_symlink() or output_root.resolve() != output_root:
raise CompositionError("node output root must be a real directory")
entries = _files(output_root)
identity = {
"schema_version": NODE_RESULT_SCHEMA,
"input_identity": input_identity,
"input_sha256": input_sha256,
"outputs": entries,
"metadata": metadata,
}
result_sha256 = canonical_sha256(identity)
payload = canonical_bytes({**identity, "result_sha256": result_sha256})
if len(payload) > _MAX_MANIFEST_BYTES:
raise CompositionError("node manifest exceeds the size limit")
stage = Path(tempfile.mkdtemp(prefix=".node-seal-", dir=self.root))
try:
data = stage / "data"
data.mkdir(mode=0o700)
for entry in entries:
relative = PurePosixPath(cast(str, entry["path"]))
target = data.joinpath(*relative.parts)
target.parent.mkdir(parents=True, exist_ok=True)
# Copy: chmod on a hardlink would mutate the producer's files,
# and a surviving producer could corrupt a supposedly sealed entry.
with (output_root / relative).open("rb") as source, target.open("xb") as out:
shutil.copyfileobj(source, out, 1024 * 1024)
out.flush()
os.fsync(out.fileno())
target.chmod(0o444)
if _files(data) != entries:
raise CompositionError("node output changed while sealing")
with (stage / "manifest.json").open("xb") as out:
out.write(payload)
out.flush()
os.fsync(out.fileno())
(stage / "manifest.json").chmod(0o444)
try:
stage.rename(destination)
except OSError:
# Another process may have atomically won this exact identity.
winner = self.lookup(input_identity)
if winner is None or winner.result_sha256 != result_sha256:
raise CompositionError("node seal conflicts with an existing result") from None
return winner
_fsync_directory(self.root)
return SealedNodeResult(destination / "data", input_sha256, result_sha256, payload)
finally:
if stage.exists():
shutil.rmtree(stage)
def _fsync_directory(path: Path) -> None:
descriptor = os.open(path, os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
+252
View File
@@ -0,0 +1,252 @@
"""Result contract for one independently selected Observatory AI module."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
from typing import Final, cast
from k1link.observatory.portable_result_contract import (
OBSERVATION_ONLY_AUTHORITY,
PortableResultArtifact,
PortableResultPackageIntegrityError,
PortableResultValidationContext,
)
from k1link.observatory.portable_run_definitions import canonical_sha256
MODULAR_RESULT_SCHEMA: Final = "missioncore.recorded-ai-layer-review/v1"
MODULAR_RESULT_KIND: Final = "recorded-ai-layer-review"
MODULAR_RESULT_CONTRACT_SHA256: Final = (
"2316c027ed6014d3ca5632e38de3bbed448ca79e7243fb300510565b21d56305"
)
MODULE_BY_SETUP: Final = {
"ai-segmentation-ddrnet-v1": "ddrnet",
"ai-segmentation-eomt-v1": "eomt",
"ai-detection-rf-detr-v1": "rf-detr",
"ai-range-object-distance-v1": "object-distance",
}
_EXPECTED_ARTIFACT_ROLES: Final = {
"ddrnet": {
"ddrnet-decode-repair",
"ddrnet-result-document",
"ddrnet-semantic-mask-archive",
},
"eomt": {
"eomt-decode-repair",
"eomt-panoptic-frame-metadata",
"eomt-gpu-telemetry",
"eomt-panoptic-mask-archive",
"eomt-overlay-video",
"eomt-result-document",
"eomt-run-report",
"eomt-source-frame-manifest",
},
"rf-detr": {"rf-detr-frame-detections", "rf-detr-result-document"},
"object-distance": {
"object-distance-frame-observations",
"object-distance-result-document",
"rf-detr-frame-detections",
"rf-detr-result-document",
},
}
def validate_modular_result(context: PortableResultValidationContext) -> None:
"""Re-bind a packaged module result to its exact job and artifacts."""
definition = context.definition
module_id = MODULE_BY_SETUP.get(definition.setup_id)
document = context.result_document
if (
module_id is None
or definition.result_contract.contract_sha256 != MODULAR_RESULT_CONTRACT_SHA256
or definition.result_contract.result_schema != MODULAR_RESULT_SCHEMA
or definition.result_contract.result_kind != MODULAR_RESULT_KIND
or set(document)
!= {
"schema_version",
"result_id",
"result_kind",
"identity",
"identity_sha256",
"source",
"module",
"artifacts",
"authority",
}
or document.get("schema_version") != MODULAR_RESULT_SCHEMA
or document.get("result_kind") != MODULAR_RESULT_KIND
or document.get("authority") != OBSERVATION_ONLY_AUTHORITY
):
raise PortableResultPackageIntegrityError("AI-layer result envelope changed")
identity = _object(document.get("identity"), "AI-layer identity")
identity_sha = document.get("identity_sha256")
if (
not isinstance(identity_sha, str)
or canonical_sha256(identity) != identity_sha
or document.get("result_id") != f"ai-layer-{module_id}-{identity_sha}"
or document.get("result_id") != context.job.result_id
):
raise PortableResultPackageIntegrityError("AI-layer result identity changed")
source = _object(document.get("source"), "AI-layer source")
module = _object(document.get("module"), "AI-layer module")
expected_source = {
"session_id": context.job.source_session_id,
"catalog_sha256": context.job.source_catalog_sha256,
"bundle_sha256": context.job.source_bundle_sha256,
"capability_manifest_sha256": context.job.source_capability_manifest_sha256,
}
if (
any(source.get(key) != value for key, value in expected_source.items())
or module.get("module_id") != module_id
or module.get("definition_sha256") != definition.definition_sha256
or identity
!= {
"job_identity_sha256": context.job.identity_sha256,
"source_bundle_sha256": context.job.source_bundle_sha256,
"definition_sha256": definition.definition_sha256,
"module_id": module_id,
"component_result_sha256": module.get("component_result_sha256"),
}
):
raise PortableResultPackageIntegrityError("AI-layer provenance changed")
declared_value = document.get("artifacts")
if not isinstance(declared_value, list):
raise PortableResultPackageIntegrityError("AI-layer artifacts are invalid")
declared = tuple(_artifact(row) for row in declared_value)
if (
tuple(item.role for item in declared) != tuple(sorted(item.role for item in declared))
or {item.role for item in declared} != _EXPECTED_ARTIFACT_ROLES[module_id]
):
raise PortableResultPackageIntegrityError("AI-layer artifacts are not canonical")
packaged = {item.role: item for item in context.manifest.artifacts}
if set(packaged) != {"result-document", *(item.role for item in declared)}:
raise PortableResultPackageIntegrityError("AI-layer package artifacts changed")
for artifact in declared:
path = context.artifact_paths.get(artifact.role)
if (
packaged.get(artifact.role) != artifact
or path is None
or _sha256(path) != artifact.sha256
):
raise PortableResultPackageIntegrityError("AI-layer artifact content changed")
result_role = f"{module_id}-result-document"
component_path = context.artifact_paths.get(result_role)
if component_path is None or _sha256(component_path) != module.get("component_result_sha256"):
raise PortableResultPackageIntegrityError("AI-layer component result changed")
try:
component = json.loads(component_path.read_bytes())
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableResultPackageIntegrityError("AI-layer component result is invalid") from exc
_validate_component(module_id, _object(component, "component result"), source, packaged)
if module_id == "object-distance":
dependency_path = context.artifact_paths.get("rf-detr-result-document")
if dependency_path is None:
raise PortableResultPackageIntegrityError("RF-DETR dependency result is missing")
try:
dependency = _object(
json.loads(dependency_path.read_bytes()), "RF-DETR dependency result"
)
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise PortableResultPackageIntegrityError(
"RF-DETR dependency result is invalid"
) from exc
_validate_rf_detr(dependency, source, packaged)
def _validate_component(
module_id: str,
result: dict[str, object],
source: dict[str, object],
packaged: dict[str, PortableResultArtifact],
) -> None:
frame_count = source.get("frame_count")
camera_input = source.get("camera_input_sha256")
if not isinstance(frame_count, int) or isinstance(frame_count, bool) or frame_count < 1:
raise PortableResultPackageIntegrityError("AI-layer frame count is invalid")
if module_id == "eomt":
rows = result.get("artifacts")
if not isinstance(rows, list):
raise PortableResultPackageIntegrityError("EoMT artifacts are invalid")
archive = next(
(
row
for row in cast(list[object], rows)
if isinstance(row, dict) and row.get("kind") == "panoptic-mask-archive"
),
None,
)
if (
result.get("schema_version") != "missioncore.recorded-perception-result/v2"
or result.get("session_id") != source.get("session_id")
or result.get("input_sha256") != camera_input
or result.get("frames_processed") != frame_count
or not isinstance(archive, dict)
or archive.get("sha256") != packaged["eomt-panoptic-mask-archive"].sha256
):
raise PortableResultPackageIntegrityError("EoMT result binding changed")
return
if module_id == "rf-detr":
_validate_rf_detr(result, source, packaged)
return
if module_id == "object-distance":
if (
result.get("schema_version")
!= "missioncore.observatory-ai-module-object-distance-result/v1"
or result.get("module_id") != "object-distance"
or result.get("source_session_id") != source.get("session_id")
or result.get("frame_count") != frame_count
or result.get("object_distances_sha256")
!= packaged["object-distance-frame-observations"].sha256
or result.get("range_estimator") != "median-camera-z-of-owned-current-points/v1"
):
raise PortableResultPackageIntegrityError("object-distance result binding changed")
return
semantics = _object(result.get("video_semantics"), "DDRNet semantics")
archive = _object(semantics.get("mask_archive"), "DDRNet mask archive")
source_row = _object(result.get("source"), "DDRNet source")
if (
result.get("schema_version") != "missioncore.lab-v1-goose-vegetation-run/v1"
or source_row.get("input_count") != frame_count
or archive.get("sha256") != packaged["ddrnet-semantic-mask-archive"].sha256
or archive.get("frame_count") != frame_count
):
raise PortableResultPackageIntegrityError("DDRNet result binding changed")
def _validate_rf_detr(
result: dict[str, object],
source: dict[str, object],
packaged: dict[str, PortableResultArtifact],
) -> None:
source_row = _object(result.get("source"), "RF-DETR source")
if (
result.get("schema_version") != "missioncore.observatory-ai-module-rf-detr-result/v1"
or result.get("module_id") != "rf-detr"
or source_row.get("session_id") != source.get("session_id")
or result.get("frame_count") != source.get("frame_count")
or result.get("detections_sha256") != packaged["rf-detr-frame-detections"].sha256
):
raise PortableResultPackageIntegrityError("RF-DETR result binding changed")
def _artifact(value: object) -> PortableResultArtifact:
row = _object(value, "AI-layer artifact")
if set(row) != {"role", "relative_path", "media_type", "byte_length", "sha256"}:
raise PortableResultPackageIntegrityError("AI-layer artifact fields changed")
try:
return PortableResultArtifact(**row) # type: ignore[arg-type]
except (TypeError, ValueError) as exc:
raise PortableResultPackageIntegrityError("AI-layer artifact is invalid") from exc
def _object(value: object, label: str) -> dict[str, object]:
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
raise PortableResultPackageIntegrityError(f"{label} is invalid")
return cast(dict[str, object], value)
def _sha256(path: Path) -> str:
with path.open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
@@ -52,12 +52,8 @@ from k1link.observatory.source_admission import (
)
from k1link.observatory.worker_agent import SealedObservatoryRecordedJob
PORTABLE_LAB_V1_SOURCE_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-source/v1"
)
PORTABLE_LAB_V1_PLAN_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-orchestration-plan/v1"
)
PORTABLE_LAB_V1_SOURCE_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-source/v1"
PORTABLE_LAB_V1_PLAN_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-orchestration-plan/v1"
PORTABLE_LAB_V1_PLAN_IDENTITY_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-orchestration-plan-identity/v1"
)
@@ -74,9 +70,7 @@ PORTABLE_LAB_V1_RELEASE_IDENTITY_SCHEMA: Final = (
PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-executor-seal/v2"
)
PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: Final = (
"missioncore.lab-v1-eomt-ddrnet-portable-profile/v2"
)
PORTABLE_LAB_V1_DDRNET_PROFILE_SCHEMA: Final = "missioncore.lab-v1-eomt-ddrnet-portable-profile/v2"
PORTABLE_LAB_V1_DDRNET_EFFECTIVE_CONFIG_SCHEMA: Final = (
"missioncore.lab-v1-goose-vegetation-benchmark/v1"
)
@@ -97,9 +91,7 @@ _DDRNET_CANDIDATE_KEY: Final = "ddrnet"
_DDRNET_CHECKPOINT_SHA256: Final = (
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
)
_GOOSE_MAPPING_SHA256: Final = (
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
)
_GOOSE_MAPPING_SHA256: Final = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
_OBSERVATORY_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$")
_CAMERA_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$")
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
@@ -214,9 +206,7 @@ class PortableLabV1SourceInput:
"session_id": self.source_session_id,
"catalog_sha256": self.source_catalog_sha256,
"bundle_sha256": self.source_bundle_sha256,
"capability_manifest_sha256": (
self.source_capability_manifest_sha256
),
"capability_manifest_sha256": (self.source_capability_manifest_sha256),
"adapter_sha256": self.source_adapter_sha256,
},
"camera_compute_job": {
@@ -341,13 +331,8 @@ def materialize_lab_v1_source_input(
)
if hashlib.sha256(source_bundle_bytes).hexdigest() != job.source_bundle_sha256:
raise PortableLabV1SourceError("source bundle digest differs from the sealed job")
if (
hashlib.sha256(capability_bytes).hexdigest()
!= job.source_capability_manifest_sha256
):
raise PortableLabV1SourceError(
"source capability digest differs from the sealed job"
)
if hashlib.sha256(capability_bytes).hexdigest() != job.source_capability_manifest_sha256:
raise PortableLabV1SourceError("source capability digest differs from the sealed job")
_validate_source_documents(
source_bundle=source_bundle,
capability=capability,
@@ -484,9 +469,7 @@ class PortableLabV1ReleaseCandidate:
_digest(self.executor_image_sha256, "release executor image sha256")
ids = tuple(asset.asset_id for asset in self.assets)
if ids != tuple(sorted(ids)) or len(ids) != len(set(ids)):
raise PortableLabV1ReleaseError(
"release assets must be unique and canonically ordered"
)
raise PortableLabV1ReleaseError("release assets must be unique and canonically ordered")
if not self.phases or len(self.phases) != len(set(self.phases)):
raise PortableLabV1ReleaseError("release phases are invalid")
for phase in self.phases:
@@ -561,9 +544,7 @@ class PortableLabV1ReleaseCandidate:
return cls(
release_id=_string(document["release_id"], "release id"),
setup_id=_string(document["setup_id"], "release setup id"),
definition_id=_string(
document["definition_id"], "release definition id"
),
definition_id=_string(document["definition_id"], "release definition id"),
definition_version=_positive_int(
document["definition_version"], "release definition version"
),
@@ -579,9 +560,7 @@ class PortableLabV1ReleaseCandidate:
assets=assets,
phases=phases,
declared_blockers=blockers,
candidate_sha256=_string(
document["candidate_sha256"], "release candidate sha256"
),
candidate_sha256=_string(document["candidate_sha256"], "release candidate sha256"),
repository_root=_real_directory(repository_root, "repository root"),
)
@@ -606,14 +585,10 @@ class PortableLabV1ReleaseCandidate:
definition.setup_id != self.setup_id
or definition.definition_id != self.definition_id
or definition.version != self.definition_version
or definition.executable_contract_sha256
!= self.definition_contract_sha256
or definition.result_contract.contract_sha256
!= self.result_contract_sha256
or definition.executable_contract_sha256 != self.definition_contract_sha256
or definition.result_contract.contract_sha256 != self.result_contract_sha256
):
raise PortableLabV1ReleaseError(
"release candidate belongs to another RunDefinition"
)
raise PortableLabV1ReleaseError("release candidate belongs to another RunDefinition")
def inspect(
self,
@@ -660,9 +635,7 @@ class PortableLabV1ReleaseCandidate:
or self.executor_image_sha256 is None
or len(inspection.matched_assets) != len(self.assets)
):
raise PortableLabV1ReleaseError(
"portable LAB V1 executor candidate is not sealable"
)
raise PortableLabV1ReleaseError("portable LAB V1 executor candidate is not sealable")
identity = {
"schema_version": PORTABLE_LAB_V1_EXECUTOR_SEAL_SCHEMA,
"release_id": self.release_id,
@@ -725,9 +698,8 @@ class PortableLabV1PlanPhase:
def __post_init__(self) -> None:
_pattern(self.phase_id, _IDENTIFIER, "plan phase id")
if (
not self.component_sha256s
or self.component_sha256s != tuple(sorted(self.component_sha256s))
if not self.component_sha256s or self.component_sha256s != tuple(
sorted(self.component_sha256s)
):
raise PortableLabV1PlanError("plan component identities are not canonical")
for digest_value in self.component_sha256s:
@@ -787,10 +759,8 @@ class PortableLabV1OrchestrationPlan:
_digest(value, label)
if (
self.source_input.observatory_job_id != self.observatory_job_id
or self.source_input.observatory_request_sha256
!= self.observatory_request_sha256
or self.source_input.observatory_identity_sha256
!= self.observatory_identity_sha256
or self.source_input.observatory_request_sha256 != self.observatory_request_sha256
or self.source_input.observatory_identity_sha256 != self.observatory_identity_sha256
):
raise PortableLabV1PlanError("plan source belongs to another job")
if self.blockers != tuple(sorted(self.blockers)) or len(self.blockers) != len(
@@ -804,10 +774,7 @@ class PortableLabV1OrchestrationPlan:
"result-v2-assembly",
):
raise PortableLabV1PlanError("combined LAB V1 phase order changed")
if (
canonical_sha256(self.effective_ddrnet_config)
!= self.effective_ddrnet_config_sha256
):
if canonical_sha256(self.effective_ddrnet_config) != self.effective_ddrnet_config_sha256:
raise PortableLabV1PlanError("effective DDRNet config digest changed")
if canonical_sha256(self.identity_document()) != self.plan_sha256:
raise PortableLabV1PlanError("orchestration plan identity changed")
@@ -830,8 +797,7 @@ class PortableLabV1OrchestrationPlan:
raise PortableLabV1PlanError("release inspection belongs to another candidate")
expected_asset_ids = tuple(asset.asset_id for asset in release.assets)
if release_inspection.ready and (
release_inspection.matched_assets != expected_asset_ids
or release_inspection.blockers
release_inspection.matched_assets != expected_asset_ids or release_inspection.blockers
):
raise PortableLabV1PlanError(
"ready release inspection does not admit every exact asset"
@@ -1147,13 +1113,8 @@ class PortableLabV1ResultAssembly:
roles = tuple(item.role for item in self.artifacts)
if roles != tuple(sorted(roles)) or len(roles) != len(set(roles)):
raise PortableLabV1ResultError("assembled artifacts are not canonical")
result_artifact = tuple(
item for item in self.artifacts if item.role == "result-document"
)
if (
len(result_artifact) != 1
or result_artifact[0].sha256 != self.result_document_sha256
):
result_artifact = tuple(item for item in self.artifacts if item.role == "result-document")
if len(result_artifact) != 1 or result_artifact[0].sha256 != self.result_document_sha256:
raise PortableLabV1ResultError("assembled result document is not bound")
@@ -1298,9 +1259,7 @@ def assemble_lab_v1_result_v2(
"session_id": plan.source_input.source_session_id,
"catalog_sha256": plan.source_input.source_catalog_sha256,
"bundle_sha256": plan.source_input.source_bundle_sha256,
"capability_manifest_sha256": (
plan.source_input.source_capability_manifest_sha256
),
"capability_manifest_sha256": (plan.source_input.source_capability_manifest_sha256),
"camera_input_sha256": plan.source_input.camera_input_sha256,
"frame_count": plan.source_input.frame_count,
"timeline_start_seconds": plan.source_input.timeline_start_seconds,
@@ -1311,9 +1270,7 @@ def assemble_lab_v1_result_v2(
"definition_id": definition.definition_id,
"version": definition.version,
"definition_sha256": definition.definition_sha256,
"result_contract_sha256": (
definition.result_contract.contract_sha256
),
"result_contract_sha256": (definition.result_contract.contract_sha256),
"release_candidate_sha256": plan.release_candidate_sha256,
"plan_sha256": plan.plan_sha256,
},
@@ -1343,9 +1300,7 @@ def assemble_lab_v1_result_v2(
)
final = parent / result_id
if final.exists():
raise PortableLabV1ResultError(
"an assembled result with this identity already exists"
)
raise PortableLabV1ResultError("an assembled result with this identity already exists")
_fsync_tree(staging)
os.replace(staging, final)
_fsync_directory(parent)
@@ -1429,8 +1384,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
definition.setup_id != _EXPECTED_SETUP_ID
or definition.definition_id != _EXPECTED_DEFINITION_ID
or definition.result_contract.result_schema != PORTABLE_LAB_V1_RESULT_SCHEMA
or definition.result_contract.contract_sha256
!= _EXPECTED_RESULT_CONTRACT_SHA256
or definition.result_contract.contract_sha256 != _EXPECTED_RESULT_CONTRACT_SHA256
):
raise PortableLabV1ResultError("LAB V1 validator received another definition")
document = dict(context.result_document)
@@ -1444,8 +1398,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
identity = _object(document["identity"], "portable LAB V1 identity")
if (
canonical_sha256(identity) != document["identity_sha256"]
or document["result_id"]
!= f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
or document["result_id"] != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
):
raise PortableLabV1ResultError("portable LAB V1 result identity changed")
source = _object(document["source"], "portable LAB V1 source")
@@ -1484,8 +1437,7 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
source.get("session_id") != context.job.source_session_id
or source.get("catalog_sha256") != context.job.source_catalog_sha256
or source.get("bundle_sha256") != context.job.source_bundle_sha256
or source.get("capability_manifest_sha256")
!= context.job.source_capability_manifest_sha256
or source.get("capability_manifest_sha256") != context.job.source_capability_manifest_sha256
or run_definition.get("setup_id") != definition.setup_id
or run_definition.get("definition_id") != definition.definition_id
or run_definition.get("version") != definition.version
@@ -1499,13 +1451,10 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
raise PortableLabV1ResultError("portable LAB V1 artifacts are not an array")
declared = tuple(_artifact_from_document(row) for row in artifact_rows)
declared_roles = tuple(artifact.role for artifact in declared)
if (
declared_roles != tuple(sorted(declared_roles))
or len(declared_roles) != len(set(declared_roles))
if declared_roles != tuple(sorted(declared_roles)) or len(declared_roles) != len(
set(declared_roles)
):
raise PortableLabV1ResultError(
"portable result artifacts are not canonical"
)
raise PortableLabV1ResultError("portable result artifacts are not canonical")
declared_by_role = {artifact.role: artifact for artifact in declared}
package_by_role = {artifact.role: artifact for artifact in context.manifest.artifacts}
if set(package_by_role) != {*declared_by_role, "result-document"}:
@@ -1540,15 +1489,13 @@ def validate_lab_v1_result_v2(context: PortableResultValidationContext) -> None:
if (
source_input.identity_sha256 != identity.get("source_input_sha256")
or plan.plan_sha256 != run_definition.get("plan_sha256")
or plan.release_candidate_sha256
!= run_definition.get("release_candidate_sha256")
or plan.release_candidate_sha256 != run_definition.get("release_candidate_sha256")
or plan.observatory_job_id != context.job.job_id
or plan.observatory_request_sha256 != context.job.request_sha256
or plan.observatory_identity_sha256 != context.job.identity_sha256
or source.get("camera_input_sha256") != source_input.camera_input_sha256
or source.get("frame_count") != source_input.frame_count
or source.get("timeline_start_seconds")
!= source_input.timeline_start_seconds
or source.get("timeline_start_seconds") != source_input.timeline_start_seconds
or source.get("timeline_end_seconds") != source_input.timeline_end_seconds
):
raise PortableLabV1ResultError("portable source or plan artifact changed")
@@ -1621,9 +1568,7 @@ def portable_lab_v1_source_input_from_document(
"portable camera compute job",
)
return PortableLabV1SourceInput(
observatory_job_id=_string(
observatory_job["job_id"], "portable source Observatory job id"
),
observatory_job_id=_string(observatory_job["job_id"], "portable source Observatory job id"),
observatory_request_sha256=_string(
observatory_job["request_sha256"],
"portable source Observatory request sha256",
@@ -1633,23 +1578,15 @@ def portable_lab_v1_source_input_from_document(
"portable source Observatory identity sha256",
),
source_session_id=_string(source["session_id"], "portable source session id"),
source_catalog_sha256=_string(
source["catalog_sha256"], "portable source catalog sha256"
),
source_bundle_sha256=_string(
source["bundle_sha256"], "portable source bundle sha256"
),
source_catalog_sha256=_string(source["catalog_sha256"], "portable source catalog sha256"),
source_bundle_sha256=_string(source["bundle_sha256"], "portable source bundle sha256"),
source_capability_manifest_sha256=_string(
source["capability_manifest_sha256"],
"portable source capability sha256",
),
source_adapter_sha256=_string(
source["adapter_sha256"], "portable source adapter sha256"
),
source_adapter_sha256=_string(source["adapter_sha256"], "portable source adapter sha256"),
camera_job_id=_string(camera["job_id"], "portable camera job id"),
camera_input_sha256=_string(
camera["input_sha256"], "portable camera input sha256"
),
camera_input_sha256=_string(camera["input_sha256"], "portable camera input sha256"),
camera_source_id=_string(camera["source_id"], "portable camera source id"),
codec_epoch=_positive_int(camera["codec_epoch"], "portable codec epoch"),
input_byte_length=_positive_int(
@@ -1659,15 +1596,11 @@ def portable_lab_v1_source_input_from_document(
timeline_start_seconds=_finite_float(
camera["timeline_start_seconds"], "portable timeline start"
),
timeline_end_seconds=_finite_float(
camera["timeline_end_seconds"], "portable timeline end"
),
timeline_end_seconds=_finite_float(camera["timeline_end_seconds"], "portable timeline end"),
camera_generation_sha256=_string(
camera["generation_sha256"], "portable camera generation sha256"
),
calibration_sha256=_string(
camera["calibration_sha256"], "portable calibration sha256"
),
calibration_sha256=_string(camera["calibration_sha256"], "portable calibration sha256"),
)
@@ -1758,9 +1691,7 @@ def portable_lab_v1_orchestration_plan_from_document(
raise PortableLabV1ResultError("portable plan admission changed")
phases = tuple(_plan_phase_from_document(value) for value in phases_value)
plan = PortableLabV1OrchestrationPlan(
observatory_job_id=_string(
observatory_job["job_id"], "portable plan Observatory job id"
),
observatory_job_id=_string(observatory_job["job_id"], "portable plan Observatory job id"),
observatory_request_sha256=_string(
observatory_job["request_sha256"],
"portable plan Observatory request sha256",
@@ -1770,9 +1701,7 @@ def portable_lab_v1_orchestration_plan_from_document(
"portable plan Observatory identity sha256",
),
setup_id=_string(run_definition["setup_id"], "portable plan setup id"),
definition_id=_string(
run_definition["definition_id"], "portable plan definition id"
),
definition_id=_string(run_definition["definition_id"], "portable plan definition id"),
definition_version=_positive_int(
run_definition["version"], "portable plan definition version"
),
@@ -1817,8 +1746,7 @@ def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase:
input_values = row["input_roles"]
output_values = row["output_roles"]
if not all(
isinstance(values, list)
for values in (component_values, input_values, output_values)
isinstance(values, list) for values in (component_values, input_values, output_values)
):
raise PortableLabV1ResultError("portable plan phase arrays changed")
return PortableLabV1PlanPhase(
@@ -1828,12 +1756,10 @@ def _plan_phase_from_document(value: object) -> PortableLabV1PlanPhase:
for item in cast(list[object], component_values)
),
input_roles=tuple(
_string(item, "portable plan input role")
for item in cast(list[object], input_values)
_string(item, "portable plan input role") for item in cast(list[object], input_values)
),
output_roles=tuple(
_string(item, "portable plan output role")
for item in cast(list[object], output_values)
_string(item, "portable plan output role") for item in cast(list[object], output_values)
),
)
@@ -1859,8 +1785,7 @@ def _validate_source_documents(
source_bundle.get("source_session_id") != job.source_session_id
or capability.get("source_session_id") != job.source_session_id
or camera_job.session_id != job.source_session_id
or source_bundle.get("source_catalog_sha256")
!= job.source_catalog_sha256
or source_bundle.get("source_catalog_sha256") != job.source_catalog_sha256
or capability.get("source_catalog_sha256") != job.source_catalog_sha256
or capability.get("source_bundle_sha256") != job.source_bundle_sha256
or capability.get("source_adapter_sha256") != job.source_adapter_sha256
@@ -1897,8 +1822,7 @@ def _validate_source_documents(
raise PortableLabV1SourceError("source camera segment count changed")
if (
video.get("source_id") != requirements.camera_source_id
or video.get("semantic_channel_id")
!= requirements.camera_semantic_channel_id
or video.get("semantic_channel_id") != requirements.camera_semantic_channel_id
or video.get("seekable") is not True
or camera_job.source_id != camera.get("public_source_id")
or camera_job.codec_epoch != epoch.get("ordinal")
@@ -1934,18 +1858,19 @@ def _validate_source_documents(
if not isinstance(files, list):
raise PortableLabV1SourceError("camera compute job file set is invalid")
file_rows = {
PurePosixPath(_string(_object(row, "camera file").get("path"), "camera file path")).name:
_object(row, "camera file")
PurePosixPath(
_string(_object(row, "camera file").get("path"), "camera file path")
).name: _object(row, "camera file")
for row in files
if PurePosixPath(
_string(_object(row, "camera file").get("path"), "camera file path")
).parent.name
in (f"epoch-{camera_job.codec_epoch}", "segments")
}
if (
init.get("sha256") != _object(file_rows.get("init.mp4"), "camera init file").get("sha256")
or init.get("byte_length")
!= _object(file_rows.get("init.mp4"), "camera init file").get("byte_length")
if init.get("sha256") != _object(file_rows.get("init.mp4"), "camera init file").get(
"sha256"
) or init.get("byte_length") != _object(file_rows.get("init.mp4"), "camera init file").get(
"byte_length"
):
raise PortableLabV1SourceError("camera init differs from the admitted source")
for index, row in enumerate(segments, start=1):
@@ -1964,11 +1889,7 @@ def _verify_definition_and_job(
job: SealedObservatoryRecordedJob,
) -> None:
if (
definition.setup_id != _EXPECTED_SETUP_ID
or definition.definition_id != _EXPECTED_DEFINITION_ID
or definition.result_contract.contract_sha256
!= _EXPECTED_RESULT_CONTRACT_SHA256
or job.setup_id != definition.setup_id
job.setup_id != definition.setup_id
or job.definition_id != definition.definition_id
or job.definition_version != definition.version
or job.definition_sha256 != definition.definition_sha256
@@ -1977,14 +1898,14 @@ def _verify_definition_and_job(
or job.source_adapter_sha256 != definition.source_adapter.contract_sha256
or job.model_release_ids != definition.learned_models
or job.resource_profile_id != definition.resource_profile.profile_id
or job.executor_identity.model_manifest_sha256
!= definition.model_manifest_sha256
or job.executor_identity.model_manifest_sha256 != definition.model_manifest_sha256
or job.executor_identity.resource_profile_sha256
!= definition.resource_profile.profile_sha256
or job.checkpoint_policy != definition.resource_profile.checkpoint_policy
or job.allowed_checkpoints != definition.resource_profile.allowed_checkpoints
or definition.authority.as_dict() != OBSERVATION_ONLY_AUTHORITY
):
raise PortableLabV1SourceError("job is not the exact portable LAB V1 definition")
raise PortableLabV1SourceError("job is not the exact portable definition")
def _verify_source_and_job(
@@ -2008,8 +1929,7 @@ def _verify_source_and_job(
or source.source_session_id != job.source_session_id
or source.source_catalog_sha256 != job.source_catalog_sha256
or source.source_bundle_sha256 != job.source_bundle_sha256
or source.source_capability_manifest_sha256
!= job.source_capability_manifest_sha256
or source.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
or source.source_adapter_sha256 != job.source_adapter_sha256
or source.calibration_sha256 != requirements.calibration_identity_sha256
):
@@ -2069,8 +1989,7 @@ def _verify_plan_definition(
or plan.definition_id != definition.definition_id
or plan.definition_version != definition.version
or plan.definition_sha256 != definition.definition_sha256
or plan.result_contract_sha256
!= definition.result_contract.contract_sha256
or plan.result_contract_sha256 != definition.result_contract.contract_sha256
):
raise PortableLabV1PlanError("orchestration plan belongs to another definition")
@@ -2092,8 +2011,7 @@ def _validate_eomt_component(
or document.get("source_id") != plan.source_input.camera_source_id
or document.get("codec_epoch") != plan.source_input.codec_epoch
or document.get("timestamp_basis") != "session-time-seconds"
or document.get("timeline_start_seconds")
!= plan.source_input.timeline_start_seconds
or document.get("timeline_start_seconds") != plan.source_input.timeline_start_seconds
or document.get("timeline_end_seconds") != plan.source_input.timeline_end_seconds
or document.get("frames_processed") != plan.source_input.frame_count
or document.get("ground_truth") is not False
@@ -2186,10 +2104,8 @@ def _validate_ddrnet_component(
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
or semantics.get("outside_crop_state") != "undefined"
or semantics.get("base_m4_result_id") is not None
or provenance.get("config_sha256")
!= plan.effective_ddrnet_config_sha256
or provenance.get("policy_sha256")
!= components["vegetation-mission-policy-v1"].sha256
or provenance.get("config_sha256") != plan.effective_ddrnet_config_sha256
or provenance.get("policy_sha256") != components["vegetation-mission-policy-v1"].sha256
or provenance.get("provider_map_sha256")
!= components["vegetation-provider-label-map-v1"].sha256
or authority
@@ -2290,8 +2206,7 @@ def _validate_assembly(
if (
canonical_sha256(identity) != document["identity_sha256"]
or document["result_id"] != assembly.result_id
or document["result_id"]
!= f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
or document["result_id"] != f"lab-v1-eomt-ddrnet-{document['identity_sha256']}"
):
raise PortableLabV1ResultError("assembled result identity changed")
for artifact in assembly.artifacts:
@@ -2450,10 +2365,8 @@ def _validate_published_eomt_component(
or document.get("source_id") != plan.source_input.camera_source_id
or document.get("codec_epoch") != plan.source_input.codec_epoch
or document.get("timestamp_basis") != "session-time-seconds"
or document.get("timeline_start_seconds")
!= plan.source_input.timeline_start_seconds
or document.get("timeline_end_seconds")
!= plan.source_input.timeline_end_seconds
or document.get("timeline_start_seconds") != plan.source_input.timeline_start_seconds
or document.get("timeline_end_seconds") != plan.source_input.timeline_end_seconds
or document.get("frames_processed") != plan.source_input.frame_count
or document.get("ground_truth") is not False
or semantic.get("id") != model.model_id
@@ -2472,9 +2385,7 @@ def _validate_published_eomt_component(
}
if not isinstance(rows, list) or len(rows) != len(expected_kinds):
raise PortableLabV1ResultError("published EoMT artifact set changed")
package_by_role = {
artifact.role: artifact for artifact in context.manifest.artifacts
}
package_by_role = {artifact.role: artifact for artifact in context.manifest.artifacts}
observed_kinds: set[str] = set()
for value in rows:
row = _object(value, "published EoMT artifact")
@@ -2482,13 +2393,9 @@ def _validate_published_eomt_component(
if kind not in expected_kinds or kind in observed_kinds:
raise PortableLabV1ResultError("published EoMT artifact roles changed")
observed_kinds.add(kind)
relative = _safe_relative_path(
_string(row.get("path"), "published EoMT artifact path")
)
relative = _safe_relative_path(_string(row.get("path"), "published EoMT artifact path"))
if len(relative.parts) != 1:
raise PortableLabV1ResultError(
"published EoMT artifact path changed"
)
raise PortableLabV1ResultError("published EoMT artifact path changed")
packaged = package_by_role.get(f"eomt-{kind}")
if (
packaged is None
@@ -2496,9 +2403,7 @@ def _validate_published_eomt_component(
or row.get("byte_length") != packaged.byte_length
or row.get("sha256") != packaged.sha256
):
raise PortableLabV1ResultError(
"published EoMT artifact identity changed"
)
raise PortableLabV1ResultError("published EoMT artifact identity changed")
def _validate_published_ddrnet_component(
@@ -2557,10 +2462,8 @@ def _validate_published_ddrnet_component(
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
or semantics.get("outside_crop_state") != "undefined"
or semantics.get("base_m4_result_id") is not None
or provenance.get("config_sha256")
!= plan.effective_ddrnet_config_sha256
or provenance.get("policy_sha256")
!= components["vegetation-mission-policy-v1"].sha256
or provenance.get("config_sha256") != plan.effective_ddrnet_config_sha256
or provenance.get("policy_sha256") != components["vegetation-mission-policy-v1"].sha256
or provenance.get("provider_map_sha256")
!= components["vegetation-provider-label-map-v1"].sha256
or package_archive is None
@@ -2595,6 +2498,8 @@ def _validate_published_ddrnet_component(
"lab-v1-ravnoves-video-ddrnet-" + canonical_sha256(identity_value)
):
raise PortableLabV1ResultError("published DDRNet result identity changed")
def _model(definition: PortableRunDefinition, release_id: str): # type: ignore[no-untyped-def]
for model in definition.models:
if model.release_id == release_id:
@@ -2644,9 +2549,8 @@ def _release_binding_matches(
if path.is_symlink() or not path.is_file():
return False
return (
(asset.byte_length is None or path.stat().st_size == asset.byte_length)
and _sha256_file(path) == asset.sha256
)
asset.byte_length is None or path.stat().st_size == asset.byte_length
) and _sha256_file(path) == asset.sha256
except OSError:
return False
@@ -94,9 +94,7 @@ _EXPECTED_RESULT_CONTRACT_SHA256: Final = (
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
)
_PORTABLE_CONFIG_ASSET_ID: Final = "ddrnet-portable-config"
_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = (
"lab-v1-worker-installation-receipt"
)
_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = "lab-v1-worker-installation-receipt"
_MAX_SOURCE_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
_MAX_MATERIALIZATION_MANIFEST_BYTES: Final = 64 * 1024 * 1024
_MAX_SOURCE_MEMBERS: Final = 100_000
@@ -432,6 +430,27 @@ def materialize_lab_v1_source_from_worker_stage(
"""Build a deterministic camera job from only manifested Worker members."""
_verify_definition(definition)
return materialize_recorded_camera_source_from_worker_stage(
worker_stage=worker_stage,
job=job,
definition=definition,
output_parent=output_parent,
)
def materialize_recorded_camera_source_from_worker_stage(
*,
worker_stage: PortableWorkerSourceStage,
job: SealedObservatoryRecordedJob,
definition: PortableRunDefinition,
output_parent: Path,
) -> PortableLabV1MaterializedSource:
"""Build the shared recorded-camera input for any exact installed definition.
The historical LAB entrypoint above retains its setup-specific admission.
New modular packages use this model-neutral source preparation boundary.
"""
if (
worker_stage.source_bundle_sha256 != job.source_bundle_sha256
or worker_stage.source_capability_manifest_sha256 != job.source_capability_manifest_sha256
@@ -574,8 +593,7 @@ def _verify_candidate_release(
or candidate.definition_id != release.definition_id
or candidate.definition_version != release.definition_version
or candidate.definition_sha256 != definition.definition_sha256
or release.definition_contract_sha256
!= definition.executable_contract_sha256
or release.definition_contract_sha256 != definition.executable_contract_sha256
or candidate.result_contract_sha256 != release.result_contract_sha256
or tuple(phase.phase_id for phase in candidate.phases) != PORTABLE_LAB_V1_RUNTIME_PHASES
or executor is None
@@ -0,0 +1,237 @@
"""Project sealed RF-DETR boxes and optional K1 ranges without inference."""
from __future__ import annotations
import hashlib
import json
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import numpy as np
import rerun as rr
from k1link.artifact_gateway import CentralArtifactStore
from k1link.observatory.portable_tgs_replay import PortableReplayError, read_json, verified_file
from k1link.perception.contracts import ObjectProposal2D, ObstacleObservation
RESULT_SCHEMA = "missioncore.recorded-ai-layer-review/v1"
RF_RESULT_SCHEMA = "missioncore.observatory-ai-module-rf-detr-result/v1"
RANGE_RESULT_SCHEMA = "missioncore.observatory-ai-module-object-distance-result/v1"
RF_ROW_SCHEMA = "missioncore.observatory-ai-module-rf-detr-frame/v1"
RANGE_ROW_SCHEMA = "missioncore.observatory-ai-module-object-distance-frame/v1"
RENDERER_VERSION = "portable-rf-detr-k1-range-rerun-0.36.3-v1"
_MAX_FRAMES = 250_000
_MAX_ROW_BYTES = 8 * 1024 * 1024
@dataclass(frozen=True)
class ObjectReplayFrame:
session_seconds: float
proposals: tuple[ObjectProposal2D, ...]
ranges_m: dict[str, float | None]
@dataclass(frozen=True)
class ObjectReplayData:
frames: tuple[ObjectReplayFrame, ...]
end_seconds: float
include_ranges: bool
def _artifact(
members: dict[str, dict[str, Any]],
store: CentralArtifactStore,
role: str,
media_type: str,
) -> Path:
member = members.get(role)
if member is None or member.get("media_type") != media_type:
raise PortableReplayError("object replay artifact is missing")
return verified_file(
store.object_path(member["sha256"]), member["sha256"], member["byte_length"]
)
def _rows(path: Path, *, schema: str, count: int) -> list[dict[str, Any]]:
result: list[dict[str, Any]] = []
with path.open(encoding="utf-8") as stream:
for raw in stream:
if len(raw) > _MAX_ROW_BYTES or len(result) >= count:
raise PortableReplayError("object replay rows exceed their bound")
try:
row = json.loads(raw)
except json.JSONDecodeError as exc:
raise PortableReplayError("object replay row is invalid") from exc
if not isinstance(row, dict) or row.get("schema_version") != schema:
raise PortableReplayError("object replay row contract changed")
result.append(row)
if len(result) != count:
raise PortableReplayError("object replay frame count changed")
return result
def _time(value: object) -> float:
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise PortableReplayError("object replay time is invalid")
number = float(value)
if not math.isfinite(number):
raise PortableReplayError("object replay time is invalid")
return number
def load_object_data(
view: dict[str, Any],
store: CentralArtifactStore,
*,
source_bundle_sha256: str,
starts: list[float],
end_seconds: float,
) -> ObjectReplayData:
doc = view["result_document"]
source = doc.get("source")
module = doc.get("module")
if not isinstance(source, dict) or not isinstance(module, dict):
raise PortableReplayError("object replay result is malformed")
module_id = module.get("module_id")
if (
doc.get("schema_version") != RESULT_SCHEMA
or doc.get("result_id") != view["result_id"]
or module_id not in {"rf-detr", "object-distance"}
or source.get("session_id") != view["source_session_id"]
or source.get("bundle_sha256") != source_bundle_sha256
or not 0 < len(starts) <= _MAX_FRAMES
or source.get("frame_count") != len(starts)
or source.get("timeline_start_seconds") != starts[0]
or source.get("timeline_end_seconds") != end_seconds
or not np.isfinite([*starts, end_seconds]).all()
or np.any(np.diff([*starts, end_seconds]) <= 0)
):
raise PortableReplayError("object result and source clock disagree")
members = {row["role"]: row for row in view["artifacts"]}
if len(members) != len(view["artifacts"]):
raise PortableReplayError("object artifact roles are duplicated")
rf_path = _artifact(members, store, "rf-detr-result-document", "application/json")
rf = read_json(rf_path)
detection_path = _artifact(members, store, "rf-detr-frame-detections", "application/x-ndjson")
if (
rf.get("schema_version") != RF_RESULT_SCHEMA
or rf.get("module_id") != "rf-detr"
or not isinstance(rf.get("source"), dict)
or rf["source"].get("session_id") != view["source_session_id"]
or rf.get("frame_count") != len(starts)
or rf.get("detections_sha256") != members["rf-detr-frame-detections"]["sha256"]
or (
module_id == "rf-detr"
and module.get("component_result_sha256")
!= members["rf-detr-result-document"]["sha256"]
)
):
raise PortableReplayError("RF-DETR result binding changed")
detection_rows = _rows(detection_path, schema=RF_ROW_SCHEMA, count=len(starts))
range_rows: list[dict[str, Any]] | None = None
if module_id == "object-distance":
range_path = _artifact(
members,
store,
"object-distance-frame-observations",
"application/x-ndjson",
)
ranged = read_json(
_artifact(members, store, "object-distance-result-document", "application/json")
)
if (
ranged.get("schema_version") != RANGE_RESULT_SCHEMA
or ranged.get("module_id") != "object-distance"
or ranged.get("source_session_id") != view["source_session_id"]
or ranged.get("frame_count") != len(starts)
or ranged.get("object_distances_sha256")
!= members["object-distance-frame-observations"]["sha256"]
or module.get("component_result_sha256")
!= members["object-distance-result-document"]["sha256"]
):
raise PortableReplayError("object-distance result binding changed")
range_rows = _rows(range_path, schema=RANGE_ROW_SCHEMA, count=len(starts))
frames: list[ObjectReplayFrame] = []
try:
for index, (timestamp, row) in enumerate(zip(starts, detection_rows, strict=True)):
if row.get("frame_index") != index or _time(row.get("session_seconds")) != timestamp:
raise PortableReplayError("RF-DETR frame clock changed")
raw_proposals = row.get("proposals")
if not isinstance(raw_proposals, list):
raise PortableReplayError("RF-DETR proposals are unavailable")
proposals = tuple(ObjectProposal2D.from_dict(value) for value in raw_proposals)
proposal_ids = {item.proposal_id for item in proposals}
if len(proposal_ids) != len(proposals) or any(
item.region.x_max > 800 or item.region.y_max > 600 for item in proposals
):
raise PortableReplayError("RF-DETR proposal geometry changed")
ranges: dict[str, float | None] = {}
if range_rows is not None:
range_row = range_rows[index]
if (
range_row.get("frame_index") != index
or _time(range_row.get("session_seconds")) != timestamp
or not isinstance(range_row.get("observations"), list)
):
raise PortableReplayError("object-distance frame clock changed")
for raw in range_row["observations"]:
observation = ObstacleObservation.from_dict(raw)
distance = (
None
if observation.metric_geometry is None
else observation.metric_geometry.range_m
)
for proposal_id in observation.proposal_ids:
if proposal_id not in proposal_ids or proposal_id in ranges:
raise PortableReplayError("object-distance proposal binding changed")
ranges[proposal_id] = distance
if set(ranges) != proposal_ids:
raise PortableReplayError("object-distance coverage changed")
frames.append(ObjectReplayFrame(timestamp, proposals, ranges))
except (KeyError, TypeError, ValueError) as exc:
if isinstance(exc, PortableReplayError):
raise
raise PortableReplayError("object replay contract changed") from exc
return ObjectReplayData(tuple(frames), end_seconds, range_rows is not None)
def _color(label: str) -> list[int]:
digest = hashlib.sha256(f"mission-core-object-{label}".encode()).digest()
return [80 + digest[channel] % 160 for channel in range(3)] + [255]
def log_objects(recording: rr.RecordingStream, data: ObjectReplayData) -> None:
for frame in data.frames:
recording.set_time(
"session_time",
duration=np.timedelta64(round(frame.session_seconds * 1e9), "ns"),
)
if not frame.proposals:
recording.log("/perception/camera/detections", rr.Clear(recursive=False))
continue
labels: list[str] = []
for proposal in frame.proposals:
label = proposal.semantic_hint or "объект"
confidence = f"{proposal.objectness * 100:.0f}%"
if data.include_ranges:
distance = frame.ranges_m[proposal.proposal_id]
suffix = "дальность н/д" if distance is None else f"{distance:.1f} м"
labels.append(f"{label} · {confidence} · {suffix}")
else:
labels.append(f"{label} · {confidence}")
recording.log(
"/perception/camera/detections",
rr.Boxes2D(
array=[proposal.region.as_tuple() for proposal in frame.proposals],
array_format=rr.Box2DFormat.XYXY,
labels=labels,
colors=[_color(proposal.semantic_hint or "object") for proposal in frame.proposals],
show_labels=True,
),
)
recording.set_time("session_time", duration=np.timedelta64(round(data.end_seconds * 1e9), "ns"))
recording.log("/perception/camera/detections", rr.Clear(recursive=False))
@@ -2,7 +2,7 @@
from __future__ import annotations
from collections.abc import Callable
from collections.abc import Callable, Iterator
from contextlib import suppress
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
@@ -14,6 +14,7 @@ from .portable_artifact_transport import (
from .portable_result_contract import PortableResultPublisherError
from .portable_result_publisher import PortableObservatoryResultPublisher
from .recorded_jobs import (
ObservatoryRecordedJob,
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueError,
)
@@ -55,20 +56,19 @@ class PortablePublicationReconciler:
now = self.clock()
if now.tzinfo is None:
raise ValueError("publication reconciliation clock must be timezone-aware")
candidates = self.queue.pending_publications()[:limit]
examined = 0
published = 0
failed = 0
deferred = 0
exhausted = 0
for job in candidates:
for job in self._candidates():
examined += 1
attempts = job.publication_attempts
if attempts >= self.maximum_attempts:
exhausted += 1
continue
updated_at = _timestamp(job.updated_at_utc)
retry_at = updated_at + timedelta(
seconds=self.retry_delays_seconds[attempts]
)
retry_at = updated_at + timedelta(seconds=self.retry_delays_seconds[attempts])
if now < retry_at:
deferred += 1
continue
@@ -82,24 +82,37 @@ class PortablePublicationReconciler:
with suppress(ObservatoryRecordedQueueError):
self.queue.mark_publication_failed(job.job_id, message=message)
failed += 1
# Bound actual publication work, not the first rows of the outbox.
# Exhausted/backoff entries remain evidence but cannot indefinitely
# hide later ready results behind the same prefix on every tick.
if published + failed >= limit:
break
return PortablePublicationReconciliation(
examined=len(candidates),
examined=examined,
published=published,
failed=failed,
deferred=deferred,
exhausted=exhausted,
)
def _candidates(self) -> Iterator[ObservatoryRecordedJob]:
after: tuple[str, str] | None = None
while True:
page = self.queue.pending_publications(limit=32, after=after)
if not page:
return
yield from page
last = page[-1]
after = (last.created_at_utc, last.job_id)
if len(page) < 32:
return
def _timestamp(value: str) -> datetime:
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise ObservatoryRecordedQueueError(
"recorded publication timestamp is invalid"
) from exc
raise ObservatoryRecordedQueueError("recorded publication timestamp is invalid") from exc
if parsed.tzinfo is None:
raise ObservatoryRecordedQueueError(
"recorded publication timestamp has no timezone"
)
raise ObservatoryRecordedQueueError("recorded publication timestamp has no timezone")
return parsed
@@ -160,6 +160,19 @@ class PortableRecordedQueueBindingService:
source_service = self._source_service(portable)
return self._bind(recorded, source_service.check(source_session_id))
def prepare_check(
self,
*,
source_session_id: str,
setup_id: str,
definition_sha256: str,
) -> PortableRecordedRunPreparation:
"""Prepare missing camera metadata, then seal the usual non-persistent check."""
portable, recorded = self._resolve_definition(setup_id, definition_sha256)
source_service = self._source_service(portable)
return self._bind(recorded, source_service.prepare_check(source_session_id))
def admit(
self,
*,
+243 -9
View File
@@ -25,7 +25,18 @@ from k1link.laboratory.canonical_rerun_overlay import (
canonical_lab_replay,
canonical_recording_id,
)
from k1link.observatory.portable_object_replay import (
RENDERER_VERSION as OBJECT_RENDERER_VERSION,
)
from k1link.observatory.portable_object_replay import load_object_data, log_objects
from k1link.observatory.portable_result_view import PortableResultViewService
from k1link.observatory.portable_semantic_replay import (
RENDERER_VERSION as SEMANTIC_RENDERER_VERSION,
)
from k1link.observatory.portable_semantic_replay import (
load_semantic_data,
log_semantics,
)
from k1link.observatory.portable_tgs_replay import (
PortableReplayError,
load_tgs_data,
@@ -39,7 +50,11 @@ from k1link.sessions.media import RecordedMediaEpoch, RecordedMediaInspector
RENDERER_VERSION = "portable-tgs-camera-rerun-0.36.3-v1"
_SHA = re.compile(r"^[a-f0-9]{64}$")
_RESULT = re.compile(r"^m49-tgs-portable-review-[a-f0-9]{64}$")
_RESULT = re.compile(
r"^(?:m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance))-[a-f0-9]{64}$"
)
_COMPOSITION = re.compile(r"^ai-composition-[a-f0-9]{64}$")
_LOCK = threading.Lock()
@@ -65,7 +80,16 @@ class PortableReplayService:
if _RESULT.fullmatch(result_id) is None or _SHA.fullmatch(base_sha) is None:
raise PortableReplayError("unsupported replay identity")
view = cast(dict[str, Any], self.view.read(result_id))
identity = [RENDERER_VERSION, result_id, base_sha, view["artifact_manifest_id"]]
module = view.get("result_document", {}).get("module", {})
module_id = module.get("module_id") if isinstance(module, dict) else None
renderer = (
OBJECT_RENDERER_VERSION
if module_id in {"rf-detr", "object-distance"}
else SEMANTIC_RENDERER_VERSION
if result_id.startswith(("lab-v1-", "ai-layer-"))
else RENDERER_VERSION
)
identity = [renderer, result_id, base_sha, view["artifact_manifest_id"]]
return hashlib.sha256("\0".join(identity).encode()).hexdigest(), view
def cached(self, result_id: str, base_sha: str) -> CanonicalLabReplayArtifact | None:
@@ -141,15 +165,35 @@ class PortableReplayService:
):
raise PortableReplayError("source catalog changed since calculation")
epoch = self._camera(view["source_session_id"], bundle)
data = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
starts = [epoch.timeline_start_seconds] + [
epoch.timeline_start_seconds + part.end_time_seconds for part in epoch.segments[:-1]
]
if len(starts) != len(data.rows) or any(
abs(start - row["session_seconds"]) > 1e-8
for start, row in zip(starts, data.rows, strict=True)
):
raise PortableReplayError("camera and costmap anchor clocks disagree")
tgs = None
semantics = None
objects = None
if result_id.startswith("m49-tgs-portable-review-"):
tgs = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
if len(starts) != len(tgs.rows) or any(
abs(start - row["session_seconds"]) > 1e-8
for start, row in zip(starts, tgs.rows, strict=True)
):
raise PortableReplayError("camera and costmap anchor clocks disagree")
elif result_id.startswith(("ai-layer-rf-detr-", "ai-layer-object-distance-")):
objects = load_object_data(
view,
self.view.artifacts,
source_bundle_sha256=bundle_sha,
starts=starts,
end_seconds=epoch.timeline_end_seconds,
)
else:
semantics = load_semantic_data(
view,
self.view.artifacts,
source_bundle_sha256=bundle_sha,
starts=starts,
end_seconds=epoch.timeline_end_seconds,
)
recording_id = canonical_recording_id(base[0])
self.cache.mkdir(mode=0o700, parents=True, exist_ok=True)
if self.cache.is_symlink() or shutil.disk_usage(self.cache).free < 3 * 1024**3:
@@ -162,7 +206,12 @@ class PortableReplayService:
try:
recording.set_sinks(rr.FileSink(output, write_footer=True))
_log_video(recording, proxy, epoch)
log_tgs(recording, data, epoch.timeline_end_seconds)
if tgs is not None:
log_tgs(recording, tgs, epoch.timeline_end_seconds)
if semantics is not None:
log_semantics(recording, semantics)
if objects is not None:
log_objects(recording, objects)
recording.flush(timeout_sec=180)
finally:
recording.disconnect()
@@ -188,6 +237,191 @@ class PortableReplayService:
os.replace(staged, self.cache / f"{key}.json")
return artifact
def _composition_key(
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
) -> tuple[str, tuple[dict[str, Any], ...]]:
if _COMPOSITION.fullmatch(run_id) is None or _SHA.fullmatch(base_sha) is None:
raise PortableReplayError("unsupported composition replay identity")
if not result_ids or len(result_ids) != len(set(result_ids)):
raise PortableReplayError("composition replay members are invalid")
views = tuple(cast(dict[str, Any], self.view.read(result_id)) for result_id in result_ids)
source_ids = {view["source_session_id"] for view in views}
if len(source_ids) != 1:
raise PortableReplayError("composition replay members use different sources")
identity = [
"portable-composition-rerun-0.36.3-v1",
run_id,
base_sha,
*[f"{view['result_id']}:{view['artifact_manifest_id']}" for view in views],
]
return hashlib.sha256("\0".join(identity).encode()).hexdigest(), views
def cached_composition(
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
) -> CanonicalLabReplayArtifact | None:
key, _ = self._composition_key(run_id, result_ids, base_sha)
sidecar = self.cache / f"{key}.json"
if not sidecar.exists():
return None
metadata = read_json(sidecar, 8192)
digest = metadata.get("sha256")
name = metadata.get("file")
if (
metadata.get("key") != key
or not isinstance(digest, str)
or _SHA.fullmatch(digest) is None
or not isinstance(name, str)
or re.fullmatch(r"[a-f0-9]{64}\.replay\.rrd", name) is None
):
raise PortableReplayError("composition replay cache identity changed")
path = self.cache / name
if (
path.is_symlink()
or not path.is_file()
or path.stat().st_size != metadata["byte_length"]
):
raise PortableReplayError("composition replay cache length changed")
if not 4 <= metadata["byte_length"] <= 1024 * 1024 * 1024:
raise PortableReplayError("composition replay cache exceeds bounds")
stamp = stat_identity(path.stat())
fingerprint = (path, digest)
if self._verified.get(fingerprint) != stamp:
with path.open("rb") as stream:
if hashlib.file_digest(stream, "sha256").hexdigest() != digest:
raise PortableReplayError("composition replay cache digest changed")
self._verified[fingerprint] = stamp
while len(self._verified) > 32:
self._verified.popitem(last=False)
return CanonicalLabReplayArtifact(path, metadata["byte_length"], digest)
def prepare_composition(
self, run_id: str, result_ids: tuple[str, ...], base_sha: str
) -> CanonicalLabReplayArtifact:
with _LOCK:
cached = self.cached_composition(run_id, result_ids, base_sha)
if cached is not None:
return cached
key, views = self._composition_key(run_id, result_ids, base_sha)
source_session_id = cast(str, views[0]["source_session_id"])
base = self.recording_source(source_session_id)
if base is None or base[1] != base_sha:
raise PortableReplayError("composition source recording is not ready")
recording_id = canonical_recording_id(base[0])
epoch: RecordedMediaEpoch | None = None
tgs_rows: list[Any] = []
semantic_rows: list[Any] = []
object_rows: list[Any] = []
for view in views:
result_id = cast(str, view["result_id"])
binding = self.view.sessions.get_lab_instance(result_id)
if binding is None or binding.source_session_id != source_session_id:
raise PortableReplayError("composition result binding disappeared")
source = binding.provenance["source"]
bundle_sha = source["bundle_sha256"]
if not isinstance(bundle_sha, str) or _SHA.fullmatch(bundle_sha) is None:
raise PortableReplayError("composition source bundle identity is invalid")
bundle_path = (
self.data_dir / PORTABLE_SOURCE_DOCUMENT_DIRECTORY / f"{bundle_sha}.json"
)
bundle = read_json(bundle_path, 32 * 1024 * 1024)
verified_file(bundle_path, bundle_sha, bundle_path.stat().st_size)
if bundle["source_session_id"] != source_session_id:
raise PortableReplayError("composition member belongs to another session")
_, catalog_sha = self.view.sessions.get_session_with_catalog_snapshot(
source_session_id
)
if (
catalog_sha != source["catalog_sha256"]
or catalog_sha != bundle["source_catalog_sha256"]
):
raise PortableReplayError("composition source catalog changed")
member_epoch = self._camera(source_session_id, bundle)
if epoch is None:
epoch = member_epoch
elif (
epoch.timeline_start_seconds != member_epoch.timeline_start_seconds
or epoch.timeline_end_seconds != member_epoch.timeline_end_seconds
or len(epoch.segments) != len(member_epoch.segments)
):
raise PortableReplayError("composition member clocks disagree")
starts = [member_epoch.timeline_start_seconds] + [
member_epoch.timeline_start_seconds + part.end_time_seconds
for part in member_epoch.segments[:-1]
]
if result_id.startswith("m49-tgs-portable-review-"):
data = load_tgs_data(view, self.view.artifacts, source_bundle_sha256=bundle_sha)
if len(starts) != len(data.rows) or any(
abs(start - row["session_seconds"]) > 1e-8
for start, row in zip(starts, data.rows, strict=True)
):
raise PortableReplayError("composition camera and TGS clocks disagree")
tgs_rows.append(data)
elif result_id.startswith(("ai-layer-rf-detr-", "ai-layer-object-distance-")):
object_rows.append(
load_object_data(
view,
self.view.artifacts,
source_bundle_sha256=bundle_sha,
starts=starts,
end_seconds=member_epoch.timeline_end_seconds,
)
)
else:
semantic_rows.append(
load_semantic_data(
view,
self.view.artifacts,
source_bundle_sha256=bundle_sha,
starts=starts,
end_seconds=member_epoch.timeline_end_seconds,
)
)
if epoch is None:
raise PortableReplayError("composition has no replay clock")
self.cache.mkdir(mode=0o700, parents=True, exist_ok=True)
if self.cache.is_symlink() or shutil.disk_usage(self.cache).free < 3 * 1024**3:
raise PortableReplayError("replay cache has insufficient safe space")
with tempfile.TemporaryDirectory(
prefix=".pack-composition-", dir=self.cache
) as temporary:
root = Path(temporary)
proxy = self._video(epoch, root)
output = root / "overlay.rrd"
recording = rr.RecordingStream(APPLICATION_ID, recording_id=recording_id)
try:
recording.set_sinks(rr.FileSink(output, write_footer=True))
_log_video(recording, proxy, epoch)
for data in tgs_rows:
log_tgs(recording, data, epoch.timeline_end_seconds)
for data in semantic_rows:
log_semantics(recording, data)
for data in object_rows:
log_objects(recording, data)
recording.flush(timeout_sec=180)
finally:
recording.disconnect()
with output.open("rb") as stream:
digest = hashlib.file_digest(stream, "sha256").hexdigest()
artifact = canonical_lab_replay(
base[0],
base_generation_sha256=base_sha,
overlay=CanonicalLabOverlayArtifact(output, output.stat().st_size, digest),
result_id=run_id,
recording_id=recording_id,
cache_root=self.cache,
)
metadata = {
"key": key,
"file": artifact.path.name,
"sha256": artifact.sha256,
"byte_length": artifact.byte_length,
}
staged = self.cache / f".{key}.json"
staged.write_text(json.dumps(metadata, sort_keys=True), encoding="utf-8")
os.chmod(staged, 0o600)
os.replace(staged, self.cache / f"{key}.json")
return artifact
def _camera(self, session_id: str, bundle: dict[str, Any]) -> RecordedMediaEpoch:
camera = bundle["camera"]
artifact = self.view.sessions.get_recorded_media(session_id, camera["artifact_id"])
@@ -0,0 +1,329 @@
"""Project sealed LAB V1 masks, one native frame at a time, without inference.
No archive LAB, model installation, or Worker is consulted. The source camera
clock and the published component identities are the only evidence inputs.
"""
from __future__ import annotations
import hashlib
import io
import json
import re
import tarfile
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any, cast
import numpy as np
import rerun as rr
from PIL import Image
from k1link.artifact_gateway import CentralArtifactStore
from k1link.laboratory.canonical_rerun_overlay import (
_encoded_semantic_png,
_localized_semantic_label,
_semantic_palette,
)
from k1link.observatory.portable_tgs_replay import PortableReplayError, read_json, verified_file
RESULT_SCHEMA = "missioncore.recorded-eomt-ddrnet-review/v2"
RENDERER_VERSION = "portable-eomt-ddrnet-camera-rerun-0.36.3-v1"
# The target labels belong to this exact admitted EoMT preprocessing profile,
# not to the model's raw Cityscapes label order. A new profile needs an adapter.
EOMT_PROFILE_SHA = "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
EOMT_LABELS = (
"outside_valid_fov",
"person",
"bicycle",
"motorcycle",
"car",
"heavy_vehicle",
"building_structure",
"paved_road",
"sidewalk_curb",
"ground_dirt",
"grass_low_vegetation",
"tree_woody_vegetation",
"sky",
"static_obstacle",
"animal",
"other_background",
)
_PNG_BOUND = 1024 * 1024
_MAX_FRAMES = 250_000
@dataclass(frozen=True)
class SemanticReplayData:
times: tuple[float, ...]
end_seconds: float
city_masks: Path | None
vegetation_masks: Path | None
classes: dict[str, list[dict[str, Any]]]
def load_semantic_data(
view: dict[str, Any],
store: CentralArtifactStore,
*,
source_bundle_sha256: str,
starts: list[float],
end_seconds: float,
) -> SemanticReplayData:
doc = view["result_document"]
source = doc["source"]
if (
doc.get("schema_version")
not in {
RESULT_SCHEMA,
"missioncore.recorded-ai-layer-review/v1",
}
or doc.get("result_id") != view["result_id"]
or source.get("session_id") != view["source_session_id"]
or source.get("bundle_sha256") != source_bundle_sha256
or not 0 < len(starts) <= _MAX_FRAMES
or source.get("frame_count") != len(starts)
or source.get("timeline_start_seconds") != starts[0]
or source.get("timeline_end_seconds") != end_seconds
or not np.isfinite([*starts, end_seconds]).all()
or np.any(np.diff([*starts, end_seconds]) <= 0)
):
raise PortableReplayError("semantic result and source clock disagree")
members = {row["role"]: row for row in view["artifacts"]}
if len(members) != len(view["artifacts"]):
raise PortableReplayError("semantic artifact roles are duplicated")
def artifact(role: str, media: str) -> Path:
member = members.get(role)
if member is None:
raise PortableReplayError("semantic artifact is missing")
if member["media_type"] != media:
raise PortableReplayError("semantic artifact media type changed")
return verified_file(
store.object_path(member["sha256"]),
member["sha256"],
member["byte_length"],
)
schema = doc["schema_version"]
module = doc.get("module")
module_id = module.get("module_id") if isinstance(module, dict) else None
include_eomt = schema == RESULT_SCHEMA or module_id == "eomt"
include_ddrnet = schema == RESULT_SCHEMA or module_id == "ddrnet"
if not include_eomt and not include_ddrnet:
raise PortableReplayError("semantic AI module is unsupported")
city_classes: list[dict[str, Any]] = []
city_masks: Path | None = None
if include_eomt:
city = read_json(artifact("eomt-result-document", "application/json"))
if schema == RESULT_SCHEMA:
bound = doc["components"]["eomt"]
component_bound = (
bound["result_id"] == city["result_id"]
and bound["frames_processed"] == len(starts)
and bound["result_document_sha256"] == members["eomt-result-document"]["sha256"]
)
else:
component_bound = (
doc["module"].get("component_result_sha256")
== members["eomt-result-document"]["sha256"]
)
if not component_bound or (
city["identity"]["configuration"].get("profile_sha256") != EOMT_PROFILE_SHA
or city.get("frames_processed") != len(starts)
or city.get("session_id") != view["source_session_id"]
or city.get("input_sha256") != source["camera_input_sha256"]
or city.get("timestamp_basis") != "session-time-seconds"
):
raise PortableReplayError("EoMT target profile or source changed")
for class_id, label in enumerate(EOMT_LABELS):
color = hashlib.sha256(f"mission-core-segment-{class_id}".encode()).digest()
city_classes.append(
{
"class_id": class_id,
"label": label,
"color_rgb": [64 + color[channel] % 176 for channel in range(3)],
}
)
frames = artifact("eomt-panoptic-frame-metadata", "application/x-ndjson")
with frames.open("rb") as stream:
for index, timestamp in enumerate(starts):
line = stream.readline(_PNG_BOUND + 1)
if not line or len(line) > _PNG_BOUND:
raise PortableReplayError("semantic frame metadata is truncated or oversized")
row = json.loads(line)
value = row.get("session_seconds")
if (
row.get("frame_index") != index
or row.get("sequence") != index + 1
or type(value) not in (int, float)
or not np.isfinite(value)
or abs(value - timestamp) > 1e-8
):
raise PortableReplayError("semantic frame clock disagrees with camera")
for category in row["semantic_classes"]:
class_id = category.get("id")
if (
type(class_id) is not int
or not 0 < class_id < len(EOMT_LABELS)
or category.get("label") != EOMT_LABELS[class_id]
):
raise PortableReplayError("EoMT metadata taxonomy changed")
if stream.read(1):
raise PortableReplayError("semantic metadata contains extra frames")
city_masks = artifact("eomt-panoptic-mask-archive", "application/gzip")
vegetation_classes: list[dict[str, Any]] = []
vegetation_masks: Path | None = None
if include_ddrnet:
vegetation = read_json(artifact("ddrnet-result-document", "application/json"))
if schema == RESULT_SCHEMA:
bound = doc["components"]["ddrnet"]
component_bound = (
bound["result_id"] == vegetation["result_id"]
and bound["frames_processed"] == len(starts)
and bound["result_document_sha256"] == members["ddrnet-result-document"]["sha256"]
)
else:
component_bound = (
doc["module"].get("component_result_sha256")
== members["ddrnet-result-document"]["sha256"]
)
semantic = vegetation["video_semantics"]
taxonomy = semantic["taxonomy"]
vegetation_classes = taxonomy["classes"]
archive = semantic["mask_archive"]
packaged = members.get("ddrnet-semantic-mask-archive")
if (
not component_bound
or packaged is None
or taxonomy.get("schema_version") != "missioncore.lab-v1-vegetation-taxonomy/v1"
or not isinstance(vegetation_classes, list)
or {row["class_id"] for row in vegetation_classes} != set(range(64))
or archive.get("frame_count") != len(starts)
or (archive.get("width"), archive.get("height")) != (800, 600)
or archive.get("encoding") != "uint8-class-id-png"
or any(
archive.get(key) != packaged[key] for key in ("sha256", "byte_length", "media_type")
)
):
raise PortableReplayError("DDRNet mask taxonomy or archive binding changed")
_semantic_palette(cast(list[object], vegetation_classes))
for row in vegetation_classes:
if not isinstance(row.get("label"), str) or not row["label"]:
raise PortableReplayError("DDRNet class label is invalid")
vegetation_masks = artifact("ddrnet-semantic-mask-archive", "application/zip")
return SemanticReplayData(
tuple(starts),
end_seconds,
city_masks,
vegetation_masks,
{
**({"city": city_classes} if include_eomt else {}),
**({"vegetation": vegetation_classes} if include_ddrnet else {}),
},
)
def log_semantics(recording: rr.RecordingStream, data: SemanticReplayData) -> None:
try:
_log_semantics(recording, data)
except (tarfile.TarError, zipfile.BadZipFile, EOFError) as exc:
raise PortableReplayError("semantic mask archive is damaged") from exc
def _log_semantics(recording: rr.RecordingStream, data: SemanticReplayData) -> None:
"""No whole-route decoded tensor or extracted archive; release each frame."""
palettes = {
layer: _semantic_palette(cast(list[object], classes))
for layer, classes in data.classes.items()
}
for layer, classes in data.classes.items():
recording.log(
f"/perception/camera/segmentation/{layer}",
rr.AnnotationContext(
[
rr.ClassDescription(
info=rr.AnnotationInfo(
id=row["class_id"],
label=_localized_semantic_label(row["label"]),
color=[*row["color_rgb"], 255],
)
)
for row in classes
]
),
static=True,
)
def frame(layer: str, index: int, payload: bytes) -> None:
with Image.open(io.BytesIO(payload)) as image:
if image.size != (800, 600) or image.mode not in ("L", "P") or image.format != "PNG":
raise PortableReplayError("semantic mask raster changed")
mask = np.asarray(image, dtype=np.uint8)
if int(mask.max()) >= len(data.classes[layer]):
raise PortableReplayError("semantic mask contains an unknown class")
encoded = _encoded_semantic_png(mask, palettes[layer])
recording.set_time(
"session_time",
duration=np.timedelta64(round(data.times[index] * 1e9), "ns"),
)
recording.log(
f"/perception/camera/segmentation/{layer}",
rr.EncodedImage(contents=encoded, media_type="image/png", opacity=0.72, draw_order=1.0),
)
if data.city_masks is not None:
seen: set[int] = set()
root_seen = False
# Stream gzip once. Members may arrive in archive order rather than frame
# order; every log uses its own verified camera timestamp.
with tarfile.open(data.city_masks, "r|gz") as archive:
for member in archive:
if member.isdir() and member.name.rstrip("/") == "semantic-masks":
if root_seen or member.size != 0:
raise PortableReplayError("EoMT mask archive directory is invalid")
root_seen = True
cast(Any, archive).members.clear()
continue
match = re.fullmatch(r"semantic-masks/frame-([0-9]{6})\.png", member.name)
if not match or not member.isfile() or not 0 < member.size <= _PNG_BOUND:
raise PortableReplayError("EoMT mask archive member is unsafe")
index = int(match[1]) - 1
if not 0 <= index < len(data.times) or index in seen:
raise PortableReplayError(
"EoMT mask sequence is duplicated or outside coverage"
)
source = archive.extractfile(member)
if source is None:
raise PortableReplayError("EoMT mask is unavailable")
with source:
payload = source.read(_PNG_BOUND + 1)
if len(payload) != member.size:
raise PortableReplayError("EoMT mask is truncated")
frame("city", index, payload)
seen.add(index)
# tarfile otherwise retains a TarInfo for every frame even in
# streaming mode on Python 3.12. No lookup uses that history.
cast(Any, archive).members.clear()
if len(seen) != len(data.times):
raise PortableReplayError("EoMT mask sequence is incomplete")
if data.vegetation_masks is not None:
with zipfile.ZipFile(data.vegetation_masks) as masks:
names = masks.namelist()
expected = {f"masks/frame-{index + 1:06d}.png" for index in range(len(data.times))}
if len(names) != len(expected) or set(names) != expected:
raise PortableReplayError("DDRNet mask sequence is incomplete or duplicated")
for index in range(len(data.times)):
info = masks.getinfo(f"masks/frame-{index + 1:06d}.png")
if not 0 < info.file_size <= _PNG_BOUND:
raise PortableReplayError("DDRNet mask exceeds the frame bound")
frame("vegetation", index, masks.read(info))
recording.set_time(
"session_time",
duration=np.timedelta64(round(data.end_seconds * 1e9), "ns"),
)
recording.log("/perception/camera/segmentation", rr.Clear(recursive=True))
@@ -36,7 +36,15 @@ PORTABLE_LABORATORY_SETUP_CATALOG_SCHEMA: Final = (
PORTABLE_LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1"
PORTABLE_LAB_V1_DISPLAY_NAME: Final = "LAB V1 · EoMT Cityscapes Large 1024 + DDRNet-39"
PORTABLE_M49_SETUP_ID: Final = "m49-tgs-portable-v2"
PORTABLE_M49_DISPLAY_NAME: Final = "M4.9T5 · TRAVEL TGS · CPU-only, без ML"
PORTABLE_M49_DISPLAY_NAME: Final = "TRAVEL TGS"
AI_DDRNET_SETUP_ID: Final = "ai-segmentation-ddrnet-v1"
AI_EOMT_SETUP_ID: Final = "ai-segmentation-eomt-v1"
AI_RF_DETR_SETUP_ID: Final = "ai-detection-rf-detr-v1"
AI_OBJECT_DISTANCE_SETUP_ID: Final = "ai-range-object-distance-v1"
AI_DDRNET_DISPLAY_NAME: Final = "DDRNet-39 · GOOSE"
AI_EOMT_DISPLAY_NAME: Final = "EoMT Large · Cityscapes"
AI_RF_DETR_DISPLAY_NAME: Final = "RF-DETR Large"
AI_OBJECT_DISTANCE_DISPLAY_NAME: Final = "Дистанция до объектов · K1 LiDAR"
_SOURCE_ID: Final = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
_OBSERVATION_ONLY_AUTHORITY: Final = {
@@ -63,6 +71,38 @@ _SETUP_PRESENTATION: Final = {
"incompatible": "Запись не соответствует требованиям TRAVEL TGS.",
"executor_unavailable": "Переносимый вычислительный контур M4.9T5 пока недоступен.",
},
AI_DDRNET_SETUP_ID: {
"lab_id": "LAB AI-DDRNET",
"display_name": AI_DDRNET_DISPLAY_NAME,
"description": "Независимая семантическая сегментация DDRNet записанной K1-сессии.",
"compatible": "Запись соответствует требованиям DDRNet.",
"incompatible": "Запись не соответствует требованиям DDRNet.",
"executor_unavailable": "AI-модуль DDRNet на Worker 006 недоступен.",
},
AI_EOMT_SETUP_ID: {
"lab_id": "LAB AI-EOMT",
"display_name": AI_EOMT_DISPLAY_NAME,
"description": "Независимая семантическая сегментация EoMT записанной K1-сессии.",
"compatible": "Запись соответствует требованиям EoMT.",
"incompatible": "Запись не соответствует требованиям EoMT.",
"executor_unavailable": "AI-модуль EoMT на Worker 006 недоступен.",
},
AI_RF_DETR_SETUP_ID: {
"lab_id": "LAB AI-RF-DETR",
"display_name": AI_RF_DETR_DISPLAY_NAME,
"description": "Независимая детекция объектов RF-DETR на записанном видео.",
"compatible": "Запись соответствует требованиям RF-DETR.",
"incompatible": "Запись не соответствует требованиям RF-DETR.",
"executor_unavailable": "AI-модуль RF-DETR на Worker 006 недоступен.",
},
AI_OBJECT_DISTANCE_SETUP_ID: {
"lab_id": "LAB AI-RANGE",
"display_name": AI_OBJECT_DISTANCE_DISPLAY_NAME,
"description": ("RF-DETR и синхронное облако точек для оценки дистанции до объектов."),
"compatible": "Запись содержит видео, облако точек, позу и калибровку.",
"incompatible": "Для дистанции не хватает видео или пространственных данных.",
"executor_unavailable": "AI-модуль дистанции на Worker 006 недоступен.",
},
}
_MODEL_PRESENTATION: Final = {
@@ -21,6 +21,10 @@ from k1link.observatory.m49_portable_result import (
M49_PORTABLE_RESULT_CONTRACT_SHA256,
validate_m49_portable_result,
)
from k1link.observatory.modular_result import (
MODULAR_RESULT_CONTRACT_SHA256,
validate_modular_result,
)
from k1link.observatory.portable_artifact_transport import (
PortableArtifactTransportError,
PortableObservatoryArtifactTransport,
@@ -55,9 +59,7 @@ OBSERVATORY_WORKER_SOURCE_CAS_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_
OBSERVATORY_WORKER_RESULT_STAGING_ROOT_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_RESULT_STAGING_ROOT"
)
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
)
OBSERVATORY_WORKER_LOCAL_ENABLED_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_LOCAL_ENABLED"
class PortableWorkerIntegrationError(RuntimeError):
@@ -162,6 +164,10 @@ _BUILTIN_VALIDATOR_REGISTRATIONS: Final = (
contract_sha256=M49_PORTABLE_RESULT_CONTRACT_SHA256,
validator=validate_m49_portable_result,
),
PortableResultContractValidatorRegistration(
contract_sha256=MODULAR_RESULT_CONTRACT_SHA256,
validator=validate_modular_result,
),
)
_BUILTIN_VALIDATORS_BY_CONTRACT: Final = {
registration.contract_sha256: registration.validator
@@ -172,26 +178,23 @@ _BUILTIN_VALIDATORS_BY_CONTRACT: Final = {
def portable_result_validator_registry(
definitions: PortableRunDefinitionRegistry,
*,
registrations: tuple[PortableResultContractValidatorRegistration, ...]
| None = None,
registrations: tuple[PortableResultContractValidatorRegistration, ...] | None = None,
) -> PortableResultContractValidatorRegistry:
"""Select server-installed validators by exact result-contract identity."""
installed = (
_BUILTIN_VALIDATOR_REGISTRATIONS
if registrations is None
else registrations
)
installed = _BUILTIN_VALIDATOR_REGISTRATIONS if registrations is None else registrations
by_contract = {
registration.contract_sha256: registration
for registration in PortableResultContractValidatorRegistry(installed).registrations
}
selected = tuple(
by_contract[definition.result_contract.contract_sha256]
for definition in definitions.definitions
if definition.result_contract.contract_sha256 in by_contract
)
registry = PortableResultContractValidatorRegistry(selected)
selected: list[PortableResultContractValidatorRegistration] = []
selected_contracts: set[str] = set()
for definition in definitions.definitions:
contract_sha256 = definition.result_contract.contract_sha256
if contract_sha256 in by_contract and contract_sha256 not in selected_contracts:
selected.append(by_contract[contract_sha256])
selected_contracts.add(contract_sha256)
registry = PortableResultContractValidatorRegistry(tuple(selected))
for definition in definitions.definitions:
if definition.executor.ready:
try:
@@ -287,8 +290,7 @@ def build_portable_observatory_worker_integration(
definition.setup_id
for definition in definitions.definitions
if any(
registration.contract_sha256
== definition.result_contract.contract_sha256
registration.contract_sha256 == definition.result_contract.contract_sha256
for registration in validator_registry.registrations
)
),
+81 -66
View File
@@ -575,9 +575,7 @@ class ObservatoryRecordedJob:
self.claim_heartbeat_at_utc,
)
if (self.active_claim_token is None) != (self.active_claimant_id is None):
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job claim ownership is partial"
)
raise ObservatoryRecordedQueueIntegrityError("recorded-job claim ownership is partial")
if self.active_claim_token is None:
if any(value is not None for value in lease_values) or self.claim_renewal_count != 0:
raise ObservatoryRecordedQueueIntegrityError(
@@ -654,9 +652,7 @@ class ObservatoryRecordedJob:
and self.publication_error is None
and self.published_at_utc is None
),
"pending": (
self.publication_error is None and self.published_at_utc is None
),
"pending": (self.publication_error is None and self.published_at_utc is None),
"failed": (
self.publication_attempts >= 1
and self.publication_error is not None
@@ -673,9 +669,7 @@ class ObservatoryRecordedJob:
"recorded-job publication receipt is inconsistent"
)
if self.publication_state != "not-required" and (
self.state != "succeeded"
or self.result_id is None
or self.result_sha256 is None
self.state != "succeeded" or self.result_id is None or self.result_sha256 is None
):
raise ObservatoryRecordedQueueIntegrityError(
"recorded-job publication lifecycle has no execution result"
@@ -854,10 +848,7 @@ class ObservatoryRecordedReconciliationRequest:
_validate_text(self.reason, "reconciliation reason", max_length=1_000)
if self.resource_release_attestation.job_id != self.job_id:
raise ValueError("resource-release attestation is bound to another job")
if (
self.resource_release_attestation.claim_generation
!= self.expected_claim_generation
):
if self.resource_release_attestation.claim_generation != self.expected_claim_generation:
raise ValueError("resource-release attestation is bound to another generation")
@property
@@ -1237,16 +1228,15 @@ class ObservatoryRecordedJobQueue:
if reject_duplicate_computation:
# This check shares the INSERT transaction: two clients with
# different idempotency keys cannot race into duplicate jobs.
# A sealed result awaiting publication is not a reason to run
# the models again. Published-cache validity is a separate gate.
# One sealed recording/profile version is calculated once after
# it has started successfully. Failed attempts remain as audit
# evidence and may be resubmitted as a fresh queue attempt.
duplicate = connection.execute(
"SELECT job_id FROM observatory_recorded_jobs "
"WHERE identity_sha256 = ? AND ("
"state IN ('accepted', 'queued', 'claimed', 'running', 'paused', "
"'preemption-pending', 'reconciliation-required') OR "
"(state = 'succeeded' AND publication_state IN ('pending', 'failed'))) "
"WHERE source_session_id = ? AND setup_id = ? "
"AND definition_sha256 = ? AND state != 'failed' "
"ORDER BY created_at_utc DESC, job_id DESC LIMIT 1",
(identity_sha256,),
(intent.source_session_id, definition.setup_id, definition.definition_sha256),
).fetchone()
if duplicate is not None:
raise ObservatoryRecordedQueueDuplicateError(duplicate["job_id"])
@@ -1361,12 +1351,13 @@ class ObservatoryRecordedJobQueue:
grant_count = connection.execute(
"SELECT COUNT(*) FROM observatory_recorded_claim_grants_v3"
).fetchone()[0]
job_counts = dict(connection.execute(
"SELECT state, COUNT(*) FROM observatory_recorded_jobs GROUP BY state"
).fetchall())
job_counts = dict(
connection.execute(
"SELECT state, COUNT(*) FROM observatory_recorded_jobs GROUP BY state"
).fetchall()
)
live_count = connection.execute(
"SELECT COUNT(*) FROM observatory_live_leases "
"WHERE state IN ('pending', 'active')"
"SELECT COUNT(*) FROM observatory_live_leases WHERE state IN ('pending', 'active')"
).fetchone()[0]
return {
"schema_version": "missioncore.observatory-claim-readiness/v1",
@@ -1622,7 +1613,11 @@ class ObservatoryRecordedJobQueue:
return self._get_job(connection, job_id)
def report_progress(
self, job_id: str, *, claim_token: str, claimant_id: str,
self,
job_id: str,
*,
claim_token: str,
claimant_id: str,
progress: RecordedProgress,
) -> None:
"""Replace one small observation, fenced in the ownership transaction."""
@@ -1665,10 +1660,11 @@ class ObservatoryRecordedJobQueue:
job = self._get_job(connection, job_id)
row = connection.execute(
"SELECT snapshot_json, received_at_utc FROM observatory_recorded_progress "
"WHERE job_id = ?", (job_id,),
"WHERE job_id = ?",
(job_id,),
).fetchone()
progress = None if row is None else RecordedProgress.model_validate_json(
row["snapshot_json"]
progress = (
None if row is None else RecordedProgress.model_validate_json(row["snapshot_json"])
)
if progress is not None and progress.claim_generation != job.claim_generation:
progress = None
@@ -1681,10 +1677,15 @@ class ObservatoryRecordedJobQueue:
"claim_generation": job.claim_generation,
"state": job.state,
"received_at_utc": None if progress is None else row["received_at_utc"],
"age_seconds": None if progress is None else max(0.0, (
_parse_timestamp(self._timestamp(), "clock")
- _parse_timestamp(row["received_at_utc"], "progress receipt")
).total_seconds()),
"age_seconds": None
if progress is None
else max(
0.0,
(
_parse_timestamp(self._timestamp(), "clock")
- _parse_timestamp(row["received_at_utc"], "progress receipt")
).total_seconds(),
),
"progress": None if progress is None else progress.model_dump(mode="json"),
}
@@ -1720,10 +1721,7 @@ class ObservatoryRecordedJobQueue:
with self._transaction() as connection:
job = self._get_job(connection, job_id)
self._require_active_claim(job, claim_token, now=self._timestamp())
if (
job.claim_generation != claim_generation
or job.active_claimant_id != claimant_id
):
if job.claim_generation != claim_generation or job.active_claimant_id != claimant_id:
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim ownership is stale"
)
@@ -1756,8 +1754,7 @@ class ObservatoryRecordedJobQueue:
with self._transaction() as connection:
existing = connection.execute(
"SELECT * FROM observatory_recorded_reconciliations "
"WHERE reconciliation_id = ?",
"SELECT * FROM observatory_recorded_reconciliations WHERE reconciliation_id = ?",
(request.reconciliation_id,),
).fetchone()
if existing is not None:
@@ -2060,9 +2057,7 @@ class ObservatoryRecordedJobQueue:
with self._transaction() as connection:
job = self._get_job(connection, job_id)
if job.state != "succeeded":
raise ObservatoryRecordedQueueConflictError(
"recorded execution has not succeeded"
)
raise ObservatoryRecordedQueueConflictError("recorded execution has not succeeded")
if job.publication_state == "published":
return job
if job.publication_state not in {"pending", "failed"}:
@@ -2080,15 +2075,39 @@ class ObservatoryRecordedJobQueue:
)
return self._get_job(connection, job_id)
def pending_publications(self) -> tuple[ObservatoryRecordedJob, ...]:
"""Return durable outbox entries in deterministic retry order."""
def pending_publications(
self,
*,
limit: int | None = None,
after: tuple[str, str] | None = None,
) -> tuple[ObservatoryRecordedJob, ...]:
"""Read an outbox page without retaining the entire failed-job history.
The cursor uses immutable creation identity, not ``updated_at_utc``:
attempts and concurrent publication cannot move entries across pages.
The no-argument form preserves the existing internal inspection API.
"""
if limit is not None and (type(limit) is not int or not 1 <= limit <= 100):
raise ValueError("publication page limit must be within 1..100")
conditions = ["publication_state IN ('pending', 'failed')"]
parameters: list[object] = []
if after is not None:
if not isinstance(after, tuple) or len(after) != 2:
raise ValueError("publication cursor is invalid")
_validate_timestamp(after[0], "publication cursor timestamp")
_validate_pattern(after[1], _JOB_ID, "publication cursor job id")
conditions.append("(created_at_utc, job_id) > (?, ?)")
parameters.extend(after)
query = (
"SELECT * FROM observatory_recorded_jobs WHERE "
+ " AND ".join(conditions)
+ " ORDER BY created_at_utc, job_id"
)
if limit is not None:
query += " LIMIT ?"
parameters.append(limit)
with self._read_connection() as connection:
rows = connection.execute(
"SELECT * FROM observatory_recorded_jobs "
"WHERE publication_state IN ('pending', 'failed') "
"ORDER BY created_at_utc, job_id"
).fetchall()
rows = connection.execute(query, parameters).fetchall()
return tuple(_job_from_row(row) for row in rows)
def fail(
@@ -2187,8 +2206,12 @@ class ObservatoryRecordedJobQueue:
return tuple(_job_from_row(row) for row in rows)
def published_results(
self, *, source_session_id: str, source_catalog_sha256: str,
setup_id: str, definition_sha256: str,
self,
*,
source_session_id: str,
source_catalog_sha256: str,
setup_id: str,
definition_sha256: str,
) -> tuple[ObservatoryRecordedJob, ...]:
"""Exact cache candidates, never inferred from labels or a truncated job page."""
@@ -2527,11 +2550,10 @@ class ObservatoryRecordedJobQueue:
)
if exact_replay:
return job
if (
job.terminal_claim_token_sha256 != token_sha256
or job.terminal_code
in {"claim-lease-expired", "claim-lease-migration"}
):
if job.terminal_claim_token_sha256 != token_sha256 or job.terminal_code in {
"claim-lease-expired",
"claim-lease-migration",
}:
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job terminal acknowledgement is stale"
)
@@ -2618,9 +2640,7 @@ class ObservatoryRecordedJobQueue:
or _parse_timestamp(now, "queue timestamp")
>= _parse_timestamp(job.claim_expires_at_utc, "claim expiry")
):
raise ObservatoryRecordedQueueStaleClaimError(
"recorded-job claim lease expired"
)
raise ObservatoryRecordedQueueStaleClaimError("recorded-job claim lease expired")
def _recover_stale_claims(
self,
@@ -3162,10 +3182,7 @@ def _reconciliation_receipt_from_row(
evidence_sha256=row["resource_release_evidence_sha256"],
attested_at_utc=row["resource_release_attested_at_utc"],
)
if (
attestation.attestation_sha256
!= row["resource_release_attestation_sha256"]
):
if attestation.attestation_sha256 != row["resource_release_attestation_sha256"]:
raise ObservatoryRecordedQueueIntegrityError(
"stored resource-release attestation identity changed"
)
@@ -3433,9 +3450,7 @@ def _validate_claim_lease_seconds(value: object) -> None:
if (
not isinstance(value, int)
or isinstance(value, bool)
or not MIN_RECORDED_CLAIM_LEASE_SECONDS
<= value
<= MAX_RECORDED_CLAIM_LEASE_SECONDS
or not MIN_RECORDED_CLAIM_LEASE_SECONDS <= value <= MAX_RECORDED_CLAIM_LEASE_SECONDS
):
raise ValueError("recorded-job claim lease duration is invalid")
+21 -14
View File
@@ -340,6 +340,20 @@ class RecordedK1SourceAdmissionService:
prepare_media=False,
)
def prepare_check(self, source_session_id: str) -> PortableRecordedSourceAdmission:
"""Prepare the recorded-media manifest and return a non-persistent check.
This is the bounded one-click path used when a compatible recording has
not previously been opened in the viewer. Source bundle documents are
still written only by ``admit`` after the returned identity is fenced.
"""
return self._prepare(
source_session_id,
persist=False,
prepare_media=True,
)
def admit(
self,
source_session_id: str,
@@ -606,14 +620,10 @@ class RecordedK1SourceAdmissionService:
catalog_artifact = catalog_artifacts.get(replay_artifact.artifact_id)
exact_metadata_member = (
catalog_artifact is not None
and replay_artifact.artifact_id
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and catalog_artifact.kind
== PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and replay_artifact.media_type
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
and catalog_artifact.media_type
== PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
and replay_artifact.artifact_id == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and catalog_artifact.kind == PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
and replay_artifact.media_type == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
and catalog_artifact.media_type == PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
)
digest_matches_catalog = (
replay_artifact.expected_sha256 == catalog_artifact.sha256
@@ -854,13 +864,10 @@ def _seal_replay_artifact_digests(
catalog_artifact = catalog_artifacts.get(artifact.artifact_id)
if (
catalog_artifact is None
or artifact.artifact_id
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or catalog_artifact.kind
!= PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or artifact.artifact_id != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or catalog_artifact.kind != PORTABLE_SPATIAL_REPLAY_METADATA_ARTIFACT_ID
or artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
or catalog_artifact.media_type
!= PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
or catalog_artifact.media_type != PORTABLE_SPATIAL_REPLAY_METADATA_MEDIA_TYPE
or catalog_artifact.sha256 is not None
or catalog_artifact.integrity_status not in _SEALED_ARTIFACT_STATES
or artifact.file_byte_length != catalog_artifact.byte_length
+93 -16
View File
@@ -15,8 +15,10 @@ from __future__ import annotations
import hashlib
import json
import logging
import re
import threading
import time
from collections.abc import Callable, Mapping
from contextlib import nullcontext
from dataclasses import dataclass
@@ -36,6 +38,7 @@ from k1link.observatory.recorded_progress import observe_recorded_execution
WORKER_006_CONTOUR_ID: Final = "worker-006"
MAX_EXECUTOR_FAILURE_MESSAGE_LENGTH: Final = 512
_LOG = logging.getLogger(__name__)
_JOB_ID_PATTERN: Final = r"^observatory-run-[a-f0-9]{32}$"
_CLAIM_TOKEN_PATTERN: Final = r"^[a-f0-9]{64}$"
@@ -97,6 +100,10 @@ class ObservatoryWorkerClaimRejectedError(ObservatoryWorkerAgentError):
"""A transport supplied an unknown, spoofed, or corrupted claim."""
class ObservatoryWorkerTransientTransportError(ObservatoryWorkerAgentError):
"""A bounded request may be replayed with its unchanged idempotent identity."""
class ObservatoryWorkerExecutorUnavailableError(ObservatoryWorkerAgentError):
"""No local adapter matches the exact sealed executor identity."""
@@ -505,16 +512,28 @@ class ObservatoryWorkerAgent:
observe_recorded_execution(
active_job.claim_generation,
lambda snapshot: send_progress(
job_id=active_job.job_id, claim_token=claim.claim_token,
job_id=active_job.job_id,
claim_token=claim.claim_token,
progress=snapshot,
),
) if callable(send_progress) else nullcontext()
)
if callable(send_progress)
else nullcontext()
)
with observation:
result = adapter.execute(active_job)
if not isinstance(result, ObservatoryWorkerExecutionResult):
raise TypeError("executor returned an unknown result contract")
except Exception as exc:
# Do not expose exception text, request headers, paths or secrets.
# Preserve the independent executor failure even if its claim was lost.
_LOG.error(
"Recorded executor failed job=%s generation=%s error_class=%s cause_class=%s",
active_job.job_id,
active_job.claim_generation,
type(exc).__name__,
type(exc.__cause__).__name__ if exc.__cause__ else "none",
)
heartbeat.stop()
if heartbeat.failed:
return ObservatoryWorkerCycleReport(
@@ -594,6 +613,7 @@ class _ClaimHeartbeat:
claim_token: str,
interval_seconds: float,
stop_timeout_seconds: float,
clock: Callable[[], float] = time.monotonic,
) -> None:
if interval_seconds <= 0:
raise ValueError("Worker heartbeat interval must be positive")
@@ -602,6 +622,11 @@ class _ClaimHeartbeat:
self._claim_token = claim_token
self._interval_seconds = interval_seconds
self._stop_timeout_seconds = stop_timeout_seconds
self._clock = clock
self._lease_seconds = _heartbeat_lease_seconds(
job.claim_expires_at_utc,
job.claim_heartbeat_at_utc,
)
self._stop = threading.Event()
self._state_lock = threading.Lock()
self._failure: Exception | None = None
@@ -631,11 +656,20 @@ class _ClaimHeartbeat:
def _run(self) -> None:
sequence = self._job.claim_renewal_count + 1
# Local monotonic deadlines do not depend on Worker/Core clock skew.
# Server-side lease/generation checks remain the final authority.
deadline = self._clock() + self._lease_seconds
sequence_started = self._clock()
retrying = False
# Renew immediately once start has been acknowledged. Waiting a full
# interval here would assume that claim/start transport latency consumed
# none of the original lease window.
while not self._stop.is_set():
try:
if self._clock() >= deadline:
raise ObservatoryWorkerClaimRejectedError(
"Worker heartbeat lease budget expired"
)
acknowledgement = self._transport.renew_claim(
claimant_id=WORKER_006_CONTOUR_ID,
job_id=self._job.job_id,
@@ -648,25 +682,74 @@ class _ClaimHeartbeat:
expected_job=self._job,
expected_state=("claimed", "running"),
)
if (
renewed.claim_lease is None
or renewed.claim_lease.renewal_count != sequence
):
if renewed.claim_lease is None or renewed.claim_lease.renewal_count != sequence:
raise ObservatoryWorkerClaimRejectedError(
"Worker heartbeat acknowledgement changed its sequence"
)
deadline = sequence_started + _heartbeat_lease_seconds(
renewed.claim_lease.expires_at_utc,
renewed.claim_lease.heartbeat_at_utc,
)
if self._clock() >= deadline:
raise ObservatoryWorkerClaimRejectedError(
"Worker heartbeat acknowledgement expired"
)
if retrying:
_LOG.warning(
"Recorded heartbeat recovered job=%s generation=%s sequence=%s",
self._job.job_id,
self._job.claim_generation,
sequence,
)
retrying = False
sequence += 1
except ObservatoryWorkerTransientTransportError as exc:
if not retrying:
_LOG.warning(
"Recorded heartbeat retry job=%s generation=%s sequence=%s error_class=%s",
self._job.job_id,
self._job.claim_generation,
sequence,
type(exc).__name__,
)
retrying = True
remaining = deadline - self._clock()
if remaining <= 0 or self._stop.wait(
min(5.0, self._interval_seconds, max(0.0, remaining))
):
# Finishing compute during an unacknowledged renewal does
# not turn uncertain ownership into successful publication.
self._record_failure(exc)
return
continue
except Exception as exc:
self._record_failure(exc)
self._stop.set()
return
if self._stop.wait(self._interval_seconds):
return
sequence_started = self._clock()
def _record_failure(self, exc: Exception) -> None:
with self._state_lock:
if self._failure is None:
self._failure = exc
_LOG.error(
"Recorded heartbeat lost job=%s generation=%s error_class=%s cause_class=%s",
self._job.job_id,
self._job.claim_generation,
type(exc).__name__,
type(exc.__cause__).__name__ if exc.__cause__ else "none",
)
def _heartbeat_lease_seconds(expires: str | None, renewed: str | None) -> float:
if expires is None or renewed is None:
raise ObservatoryWorkerClaimRejectedError("Worker heartbeat has no lease clock")
duration = (datetime.fromisoformat(expires) - datetime.fromisoformat(renewed)).total_seconds()
if not 0 < duration <= 3600:
raise ObservatoryWorkerClaimRejectedError("Worker heartbeat lease duration is invalid")
return duration
def _validate_claim(
@@ -687,8 +770,7 @@ def _validate_claim(
"claim_request_id": claim_request_id,
"claimant_id": WORKER_006_CONTOUR_ID,
"supported_executor_identities": [
identity.as_dict()
for identity in sorted(supported_executor_identities)
identity.as_dict() for identity in sorted(supported_executor_identities)
],
}
)
@@ -699,9 +781,7 @@ def _validate_claim(
"Worker claim job is not in a claimed generation"
)
if claim.job.claim_lease is None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim has no renewable lease"
)
raise ObservatoryWorkerClaimRejectedError("Worker claim has no renewable lease")
if claim.job.result is not None or claim.job.terminal is not None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim already carries a terminal outcome"
@@ -889,12 +969,9 @@ def _default_claim_request_id() -> str:
def _default_heartbeat_interval(job: SealedObservatoryRecordedJob) -> float:
if job.claim_heartbeat_at_utc is None or job.claim_expires_at_utc is None:
raise ObservatoryWorkerClaimRejectedError(
"Worker claim has no heartbeat lease bounds"
)
raise ObservatoryWorkerClaimRejectedError("Worker claim has no heartbeat lease bounds")
remaining = (
_parse_timestamp(job.claim_expires_at_utc)
- _parse_timestamp(job.claim_heartbeat_at_utc)
_parse_timestamp(job.claim_expires_at_utc) - _parse_timestamp(job.claim_heartbeat_at_utc)
).total_seconds()
if remaining <= 0:
raise ObservatoryWorkerClaimRejectedError("Worker claim lease already expired")
+159 -236
View File
@@ -56,9 +56,11 @@ from k1link.observatory.worker_agent import (
WORKER_006_CONTOUR_ID,
ObservatoryWorkerExecutionResult,
ObservatoryWorkerExecutorIdentity,
ObservatoryWorkerTransientTransportError,
ObservatoryWorkerTransport,
SealedObservatoryRecordedJob,
)
from k1link.observatory.worker_source_cache import WorkerSourceCache, WorkerSourceCacheError
WORKER_HTTP_MAX_JSON_BYTES: Final = 16 * 1024 * 1024
WORKER_HTTP_COPY_CHUNK_BYTES: Final = 1024 * 1024
@@ -84,6 +86,13 @@ class ObservatoryWorkerHttpError(RuntimeError):
"""The authenticated Worker HTTP boundary failed closed."""
class ObservatoryWorkerHttpTransientError(
ObservatoryWorkerHttpError,
ObservatoryWorkerTransientTransportError,
):
"""Temporary unavailability, not permission to replay arbitrary operations."""
@dataclass(frozen=True, slots=True)
class _ClaimContext:
job_id: str
@@ -128,9 +137,7 @@ class _SourceDestinationLayout:
if member.primary:
return self.root / "mqtt.raw.k1mqtt"
if self.spatial is None:
raise ObservatoryWorkerHttpError(
"spatial source destination is unavailable"
)
raise ObservatoryWorkerHttpError("spatial source destination is unavailable")
return self.spatial / member.member_id
if member.kind == "spatial-replay-metadata":
return self.root / "mqtt.metadata.jsonl"
@@ -165,6 +172,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
contour_id: str = WORKER_006_CONTOUR_ID,
timeout_seconds: float = WORKER_HTTP_DEFAULT_TIMEOUT_SECONDS,
transport: httpx.BaseTransport | None = None,
source_cache_root: Path | None = None,
) -> None:
self._base_url = _validated_base_url(base_url)
if _TOKEN.fullmatch(bearer_token) is None:
@@ -180,6 +188,9 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
}
self._timeout_seconds = timeout_seconds
self._work_root = _secure_directory(work_root)
self._source_cache = WorkerSourceCache(
source_cache_root or self._work_root / "source-cas-v1"
)
self._client = httpx.Client(
base_url=self._base_url,
headers=self._headers,
@@ -217,8 +228,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"schema_version": "missioncore.observatory-worker-claim-request/v3",
"claim_request_id": claim_request_id,
"supported_executor_identities": [
identity.as_dict()
for identity in sorted(supported_executor_identities)
identity.as_dict() for identity in sorted(supported_executor_identities)
],
},
allow_empty=True,
@@ -272,6 +282,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
return self._required_json_request(
"POST",
self._job_path(job_id, "lease/renew"),
timeout_seconds=5.0,
json_body={
"schema_version": "missioncore.observatory-worker-renew-request/v1",
"claim_token": claim_token,
@@ -281,18 +292,25 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
)
def report_progress(
self, *, job_id: str, claim_token: str, progress: RecordedProgress,
self,
*,
job_id: str,
claim_token: str,
progress: RecordedProgress,
) -> None:
context = self._require_cached_claim(job_id, claim_token)
if progress.claim_generation != context.claim_generation:
raise ObservatoryWorkerHttpError("progress generation changed")
# No response body is needed. Bounded I/O cannot stall the execution thread.
with self._client.stream(
"POST", self._job_path(job_id, "progress"),
"POST",
self._job_path(job_id, "progress"),
json={
"schema_version": "missioncore.observatory-worker-progress-request/v1",
"claim_token": claim_token, "progress": progress.model_dump(mode="json"),
}, timeout=httpx.Timeout(2.0),
"claim_token": claim_token,
"progress": progress.model_dump(mode="json"),
},
timeout=httpx.Timeout(2.0),
) as response:
if response.status_code != 204:
raise ObservatoryWorkerHttpError("progress observation was not accepted")
@@ -354,7 +372,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
headers=headers,
)
members = _source_members(manifest, job)
report_recorded_progress("source-transfer", 0, len(members), "members")
report_recorded_progress("source-preparation", 0, len(members), "members")
root = _secure_directory(
self._work_root
/ "sources"
@@ -371,40 +389,56 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"source materialization members select the same local role"
)
destinations[destination] = member
ready_members: set[str] = set()
try:
for checked_members, (destination, member) in enumerate(destinations.items(), start=1):
if _matches_file(
destination, member.sha256, member.byte_length
) or self._source_cache.restore(
destination,
sha256=member.sha256,
byte_length=member.byte_length,
):
ready_members.add(member.member_id)
report_recorded_progress(
"source-preparation", checked_members, len(members), "members"
)
except WorkerSourceCacheError as exc:
raise ObservatoryWorkerHttpError("Worker source cache admission failed") from exc
camera_members = _ordered_camera_epoch_members(members)
camera_epoch_ready = all(
_matches_file(
layout.destination(member),
member.sha256,
member.byte_length,
)
for member in camera_members
)
if not camera_epoch_ready:
camera_epoch_ready = all(member.member_id in ready_members for member in camera_members)
# A partly cached epoch needs only missing members, not another complete
# epoch archive. Cold delivery retains the existing packed transport.
if not camera_epoch_ready and not any(
member.member_id in ready_members for member in camera_members
):
camera_epoch_ready = self._download_camera_epoch_archive(
job=job,
context=context,
members=camera_members,
layout=layout,
)
completed_members = len(camera_members) if camera_epoch_ready else 0
if camera_epoch_ready:
ready_members.update(member.member_id for member in camera_members)
completed_members = len(ready_members)
report_recorded_progress("source-transfer", completed_members, len(members), "members")
for destination, member in destinations.items():
if camera_epoch_ready and member.kind in {"camera-init", "camera-segment"}:
continue
if _matches_file(destination, member.sha256, member.byte_length):
completed_members += 1
report_recorded_progress(
"source-transfer", completed_members, len(members), "members",
if member.member_id not in ready_members:
self._download_member(
job_id=job.job_id,
context=context,
member=member,
destination=destination,
)
continue
self._download_member(
job_id=job.job_id,
context=context,
member=member,
destination=destination,
)
completed_members += 1
completed_members += 1
try:
self._source_cache.retain(
destination,
sha256=member.sha256,
byte_length=member.byte_length,
)
except WorkerSourceCacheError as exc:
raise ObservatoryWorkerHttpError("Worker source cache publication failed") from exc
report_recorded_progress("source-transfer", completed_members, len(members), "members")
manifest_path = root / "materialization-manifest.json"
_write_local_exact(manifest_path, canonical_json(manifest))
@@ -426,9 +460,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
try:
package = PortableResultPackageManifest.from_bytes(manifest_payload)
except Exception as exc:
raise ObservatoryWorkerHttpError(
"Worker result draft manifest is invalid"
) from exc
raise ObservatoryWorkerHttpError("Worker result draft manifest is invalid") from exc
if (
package.manifest_sha256 != draft.result_sha256
or package.result.get("result_id") != draft.result_id
@@ -441,9 +473,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"claim_generation": job.claim_generation,
}
):
raise ObservatoryWorkerHttpError(
"Worker result draft belongs to another sealed job"
)
raise ObservatoryWorkerHttpError("Worker result draft belongs to another sealed job")
headers = self._claim_headers(context)
path = self._job_path(
job.job_id,
@@ -456,9 +486,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
content=manifest_payload,
)
if plan.get("package_identity_sha256") != package.identity_sha256:
raise ObservatoryWorkerHttpError(
"result upload plan uses another package identity"
)
raise ObservatoryWorkerHttpError("result upload plan uses another package identity")
upload_members = _upload_members(plan, job, draft)
artifacts = {artifact.role: artifact for artifact in package.artifacts}
if (
@@ -478,19 +506,25 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
}
)
).hexdigest()
if artifact is None or (
artifact.media_type,
artifact.byte_length,
artifact.sha256,
) != (member.media_type, member.byte_length, member.sha256) or (
member.member_id != expected_member_id
if (
artifact is None
or (
artifact.media_type,
artifact.byte_length,
artifact.sha256,
)
!= (member.media_type, member.byte_length, member.sha256)
or (member.member_id != expected_member_id)
):
raise ObservatoryWorkerHttpError(
"result upload plan differs from the local manifest"
)
if member.uploaded:
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
"result-transfer",
member_index + 1,
len(upload_members),
"members",
)
continue
relative = relative_artifact_path(artifact.relative_path)
@@ -500,10 +534,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"PUT",
self._job_path(
job.job_id,
(
f"result-packages/{draft.result_sha256}/members/"
f"{member.member_id}"
),
(f"result-packages/{draft.result_sha256}/members/{member.member_id}"),
),
headers={**headers, "Content-Type": member.media_type},
content=stream,
@@ -514,14 +545,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
)
updated_members = _upload_members(updated, job, draft)
if not any(
item.member_id == member.member_id and item.uploaded
for item in updated_members
item.member_id == member.member_id and item.uploaded for item in updated_members
):
raise ObservatoryWorkerHttpError(
"result upload acknowledgement did not seal its member"
)
report_recorded_progress(
"result-transfer", member_index + 1, len(upload_members), "members",
"result-transfer",
member_index + 1,
len(upload_members),
"members",
)
receipt = self._required_json_request(
"POST",
@@ -536,23 +569,17 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
"job_id": job.job_id,
"job_identity_sha256": job.identity_sha256,
"claim_generation": job.claim_generation,
"claim_token_sha256": hashlib.sha256(
context.claim_token.encode("ascii")
).hexdigest(),
"claim_token_sha256": hashlib.sha256(context.claim_token.encode("ascii")).hexdigest(),
"result_id": draft.result_id,
"result_sha256": draft.result_sha256,
"package_identity_sha256": package.identity_sha256,
"member_count": len(package.artifacts),
"total_bytes": sum(
artifact.byte_length for artifact in package.artifacts
),
"total_bytes": sum(artifact.byte_length for artifact in package.artifacts),
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
}
expected_receipt = {
**receipt_identity,
"receipt_sha256": hashlib.sha256(
canonical_json(receipt_identity)
).hexdigest(),
"receipt_sha256": hashlib.sha256(canonical_json(receipt_identity)).hexdigest(),
}
if receipt != expected_receipt:
raise ObservatoryWorkerHttpError(
@@ -593,20 +620,12 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
return False
self._raise_for_status(response)
content_encoding = response.headers.get("content-encoding")
if (
content_encoding is not None
and content_encoding.lower() != "identity"
):
if content_encoding is not None and content_encoding.lower() != "identity":
raise ObservatoryWorkerHttpError(
"camera epoch archive content encoding changed"
)
if (
response.headers.get("content-type")
!= PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE
):
raise ObservatoryWorkerHttpError(
"camera epoch archive media type changed"
)
if response.headers.get("content-type") != PORTABLE_CAMERA_EPOCH_ARCHIVE_MEDIA_TYPE:
raise ObservatoryWorkerHttpError("camera epoch archive media type changed")
declared = response.headers.get("content-length")
if (
declared is None
@@ -615,9 +634,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
or len(declared) > len(str(MAX_CAMERA_EPOCH_ARCHIVE_BYTES))
or str(int(declared)) != declared
):
raise ObservatoryWorkerHttpError(
"camera epoch archive length is invalid"
)
raise ObservatoryWorkerHttpError("camera epoch archive length is invalid")
expected_bytes = int(declared)
expected_sha256 = response.headers.get(_CONTENT_SHA_HEADER, "")
archive_id = response.headers.get(
@@ -629,15 +646,10 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
or _SHA256.fullmatch(expected_sha256) is None
or _SHA256.fullmatch(archive_id) is None
):
raise ObservatoryWorkerHttpError(
"camera epoch archive seal is invalid"
)
raise ObservatoryWorkerHttpError("camera epoch archive seal is invalid")
descriptor = os.open(
temporary,
os.O_WRONLY
| os.O_CREAT
| os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0),
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
digest = hashlib.sha256()
@@ -671,9 +683,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
sha256=expected_sha256,
)
if archive_id != expected_archive_id:
raise ObservatoryWorkerHttpError(
"camera epoch archive identity changed"
)
raise ObservatoryWorkerHttpError("camera epoch archive identity changed")
_extract_camera_epoch_archive(
temporary,
layout=layout,
@@ -715,28 +725,16 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
) as response:
self._raise_for_status(response)
content_encoding = response.headers.get("content-encoding")
if (
content_encoding is not None
and content_encoding.lower() != "identity"
):
raise ObservatoryWorkerHttpError(
"source member content encoding changed"
)
if content_encoding is not None and content_encoding.lower() != "identity":
raise ObservatoryWorkerHttpError("source member content encoding changed")
declared = response.headers.get("content-length")
if declared is not None and declared != str(member.byte_length):
raise ObservatoryWorkerHttpError(
"source member Content-Length changed"
)
raise ObservatoryWorkerHttpError("source member Content-Length changed")
if response.headers.get(_CONTENT_SHA_HEADER) != member.sha256:
raise ObservatoryWorkerHttpError(
"source member digest header changed"
)
raise ObservatoryWorkerHttpError("source member digest header changed")
descriptor = os.open(
temporary,
os.O_WRONLY
| os.O_CREAT
| os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0),
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
with os.fdopen(descriptor, "wb") as stream:
@@ -751,9 +749,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
stream.flush()
os.fsync(stream.fileno())
if byte_length != member.byte_length or digest.hexdigest() != member.sha256:
raise ObservatoryWorkerHttpError(
"source member content differs from its manifest"
)
raise ObservatoryWorkerHttpError("source member content differs from its manifest")
_publish_local_file(temporary, destination, member.sha256, member.byte_length)
finally:
with suppress(FileNotFoundError):
@@ -767,6 +763,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
json_body: object | None = None,
headers: Mapping[str, str] | None = None,
content: bytes | object | None = None,
timeout_seconds: float | None = None,
) -> dict[str, object]:
payload = self._json_request(
method,
@@ -774,6 +771,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
json_body=json_body,
headers=headers,
content=content,
timeout_seconds=timeout_seconds,
)
if payload is None:
raise ObservatoryWorkerHttpError("Worker endpoint returned no document")
@@ -788,6 +786,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
headers: Mapping[str, str] | None = None,
content: bytes | object | None = None,
allow_empty: bool = False,
timeout_seconds: float | None = None,
) -> dict[str, object] | None:
try:
with self._client.stream(
@@ -796,6 +795,7 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
json=json_body,
headers=headers,
content=content, # type: ignore[arg-type]
timeout=self._client.timeout if timeout_seconds is None else timeout_seconds,
) as response:
if allow_empty and response.status_code == 204:
return None
@@ -804,11 +804,13 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
for chunk in response.iter_bytes():
payload.extend(chunk)
if len(payload) > WORKER_HTTP_MAX_JSON_BYTES:
raise ObservatoryWorkerHttpError(
"Worker JSON response exceeds bounds"
)
raise ObservatoryWorkerHttpError("Worker JSON response exceeds bounds")
except ObservatoryWorkerHttpError:
raise
except httpx.RequestError as exc:
raise ObservatoryWorkerHttpTransientError(
"Worker HTTP transport is unavailable"
) from exc
except httpx.HTTPError as exc:
raise ObservatoryWorkerHttpError("Worker HTTP transport is unavailable") from exc
if not payload:
@@ -825,6 +827,10 @@ class ObservatoryWorkerHttpGateway(ObservatoryWorkerTransport):
if response.is_redirect:
raise ObservatoryWorkerHttpError("Worker endpoint redirect was rejected")
if response.status_code < 200 or response.status_code >= 300:
if response.status_code in {408, 429, 500, 502, 503, 504}:
raise ObservatoryWorkerHttpTransientError(
f"Worker endpoint temporarily unavailable with HTTP {response.status_code}"
)
raise ObservatoryWorkerHttpError(
f"Worker endpoint rejected the request with HTTP {response.status_code}"
)
@@ -898,17 +904,13 @@ def _source_members(
or document.get("claim_generation") != job.claim_generation
or document.get("authority") != OBSERVATION_ONLY_AUTHORITY
):
raise ObservatoryWorkerHttpError(
"source materialization belongs to another sealed job"
)
raise ObservatoryWorkerHttpError("source materialization belongs to another sealed job")
source = _object(document.get("source"), "source materialization identity")
if (
set(source)
!= {"session_id", "bundle_sha256", "capability_manifest_sha256"}
set(source) != {"session_id", "bundle_sha256", "capability_manifest_sha256"}
or source.get("session_id") != job.source_session_id
or source.get("bundle_sha256") != job.source_bundle_sha256
or source.get("capability_manifest_sha256")
!= job.source_capability_manifest_sha256
or source.get("capability_manifest_sha256") != job.source_capability_manifest_sha256
):
raise ObservatoryWorkerHttpError("source materialization identity changed")
values = document.get("members")
@@ -919,8 +921,7 @@ def _source_members(
len({member.member_id for member in members}) != len(members)
or tuple(member.member_id for member in members)
!= tuple(sorted(member.member_id for member in members))
or sum(member.byte_length for member in members)
> 2 * 1024 * 1024 * 1024 * 1024
or sum(member.byte_length for member in members) > 2 * 1024 * 1024 * 1024 * 1024
):
raise ObservatoryWorkerHttpError("source materialization bounds changed")
for member in members:
@@ -941,29 +942,18 @@ def _source_members(
)
).hexdigest()
if member.member_id != expected_member_id:
raise ObservatoryWorkerHttpError(
"source materialization member identity changed"
)
bundle_members = tuple(
member for member in members if member.kind == "source-bundle"
)
capability_members = tuple(
member for member in members if member.kind == "source-capability"
)
raise ObservatoryWorkerHttpError("source materialization member identity changed")
bundle_members = tuple(member for member in members if member.kind == "source-bundle")
capability_members = tuple(member for member in members if member.kind == "source-capability")
if (
len(bundle_members) != 1
or bundle_members[0].sha256 != job.source_bundle_sha256
or len(capability_members) != 1
or capability_members[0].sha256
!= job.source_capability_manifest_sha256
or capability_members[0].sha256 != job.source_capability_manifest_sha256
):
raise ObservatoryWorkerHttpError("source materialization documents are incomplete")
if sum(
member.kind == "spatial-replay" and member.primary for member in members
) != 1:
raise ObservatoryWorkerHttpError(
"source materialization primary replay is invalid"
)
if sum(member.kind == "spatial-replay" and member.primary for member in members) != 1:
raise ObservatoryWorkerHttpError("source materialization primary replay is invalid")
metadata_members = tuple(
member for member in members if member.kind == "spatial-replay-metadata"
)
@@ -973,15 +963,9 @@ def _source_members(
or member.primary
for member in metadata_members
):
raise ObservatoryWorkerHttpError(
"source materialization replay metadata is invalid"
)
camera_inits = tuple(
member for member in members if member.kind == "camera-init"
)
camera_segments = tuple(
member for member in members if member.kind == "camera-segment"
)
raise ObservatoryWorkerHttpError("source materialization replay metadata is invalid")
camera_inits = tuple(member for member in members if member.kind == "camera-init")
camera_segments = tuple(member for member in members if member.kind == "camera-segment")
if (
len(camera_inits) != 1
or not camera_segments
@@ -991,9 +975,7 @@ def _source_members(
for member in camera_segments
)
):
raise ObservatoryWorkerHttpError(
"source materialization camera epoch is incomplete"
)
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
return members
@@ -1053,9 +1035,7 @@ def _source_member(value: object) -> _SourceMember:
or camera_epoch is not None
or camera_sequence is not None
):
raise ObservatoryWorkerHttpError(
"source document member metadata is invalid"
)
raise ObservatoryWorkerHttpError("source document member metadata is invalid")
elif kind in {"spatial-replay", "spatial-replay-metadata"}:
if (
artifact_id is None
@@ -1063,9 +1043,7 @@ def _source_member(value: object) -> _SourceMember:
or camera_sequence is not None
or (kind == "spatial-replay-metadata" and row["primary"])
):
raise ObservatoryWorkerHttpError(
"spatial source member metadata is invalid"
)
raise ObservatoryWorkerHttpError("spatial source member metadata is invalid")
elif kind == "camera-init":
if (
artifact_id is None
@@ -1073,18 +1051,9 @@ def _source_member(value: object) -> _SourceMember:
or camera_epoch is None
or camera_sequence is not None
):
raise ObservatoryWorkerHttpError(
"camera init member metadata is invalid"
)
elif (
artifact_id is None
or row["primary"]
or camera_epoch is None
or camera_sequence is None
):
raise ObservatoryWorkerHttpError(
"camera segment member metadata is invalid"
)
raise ObservatoryWorkerHttpError("camera init member metadata is invalid")
elif artifact_id is None or row["primary"] or camera_epoch is None or camera_sequence is None:
raise ObservatoryWorkerHttpError("camera segment member metadata is invalid")
return _SourceMember(
member_id=member_id,
kind=cast(
@@ -1114,27 +1083,18 @@ def _ordered_camera_epoch_members(
inits = tuple(member for member in members if member.kind == "camera-init")
segments = tuple(member for member in members if member.kind == "camera-segment")
if len(inits) != 1 or not segments:
raise ObservatoryWorkerHttpError(
"source materialization camera epoch is incomplete"
)
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
init = inits[0]
if init.artifact_id is None or init.camera_epoch is None:
raise ObservatoryWorkerHttpError(
"source materialization camera identity is incomplete"
)
ordered_segments = tuple(
sorted(segments, key=lambda member: member.camera_sequence or 0)
)
raise ObservatoryWorkerHttpError("source materialization camera identity is incomplete")
ordered_segments = tuple(sorted(segments, key=lambda member: member.camera_sequence or 0))
if tuple(member.camera_sequence for member in ordered_segments) != tuple(
range(1, len(ordered_segments) + 1)
) or any(
member.artifact_id != init.artifact_id
or member.camera_epoch != init.camera_epoch
member.artifact_id != init.artifact_id or member.camera_epoch != init.camera_epoch
for member in ordered_segments
):
raise ObservatoryWorkerHttpError(
"source materialization camera member order changed"
)
raise ObservatoryWorkerHttpError("source materialization camera member order changed")
return (init, *ordered_segments)
@@ -1147,13 +1107,9 @@ def _camera_archive_relative_name(member: _SourceMember) -> str:
def _camera_epoch_archive_byte_length(members: tuple[_SourceMember, ...]) -> int:
content_bytes = sum(
512 + ((member.byte_length + 511) // 512) * 512 for member in members
)
content_bytes = sum(512 + ((member.byte_length + 511) // 512) * 512 for member in members)
logical_bytes = content_bytes + 1024
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (
tarfile.RECORDSIZE
)
return ((logical_bytes + tarfile.RECORDSIZE - 1) // tarfile.RECORDSIZE) * (tarfile.RECORDSIZE)
def _extract_camera_epoch_archive(
@@ -1166,9 +1122,7 @@ def _extract_camera_epoch_archive(
try:
archive_metadata = archive_path.lstat()
except OSError as exc:
raise ObservatoryWorkerHttpError(
"camera epoch archive is unavailable"
) from exc
raise ObservatoryWorkerHttpError("camera epoch archive is unavailable") from exc
if (
stat.S_ISLNK(archive_metadata.st_mode)
or not stat.S_ISREG(archive_metadata.st_mode)
@@ -1178,9 +1132,7 @@ def _extract_camera_epoch_archive(
"camera epoch archive size differs from its member inventory"
)
transfer_root = _secure_directory(archive_path.parent)
staging = Path(
tempfile.mkdtemp(prefix=".camera-epoch-extract-", dir=transfer_root)
)
staging = Path(tempfile.mkdtemp(prefix=".camera-epoch-extract-", dir=transfer_root))
staged: list[Path] = []
archive_descriptor = -1
try:
@@ -1243,9 +1195,7 @@ def _extract_camera_epoch_archive(
except ObservatoryWorkerHttpError:
raise
except (OSError, tarfile.TarError) as exc:
raise ObservatoryWorkerHttpError(
"camera epoch archive could not be extracted"
) from exc
raise ObservatoryWorkerHttpError("camera epoch archive could not be extracted") from exc
finally:
if archive_descriptor >= 0:
os.close(archive_descriptor)
@@ -1261,10 +1211,7 @@ def _copy_camera_archive_member(
) -> None:
descriptor = os.open(
destination,
os.O_WRONLY
| os.O_CREAT
| os.O_EXCL
| getattr(os, "O_NOFOLLOW", 0),
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
0o600,
)
digest = hashlib.sha256()
@@ -1284,9 +1231,7 @@ def _copy_camera_archive_member(
stream.flush()
os.fsync(stream.fileno())
if byte_length != expected_byte_length or digest.hexdigest() != expected_sha256:
raise ObservatoryWorkerHttpError(
"camera epoch archive member content changed"
)
raise ObservatoryWorkerHttpError("camera epoch archive member content changed")
verified = True
finally:
if descriptor >= 0:
@@ -1300,13 +1245,9 @@ def _prepare_source_destination_layout(
root: Path,
members: tuple[_SourceMember, ...],
) -> _SourceDestinationLayout:
camera_inits = tuple(
member for member in members if member.kind == "camera-init"
)
camera_inits = tuple(member for member in members if member.kind == "camera-init")
if len(camera_inits) != 1 or camera_inits[0].camera_epoch is None:
raise ObservatoryWorkerHttpError(
"source materialization camera epoch is incomplete"
)
raise ObservatoryWorkerHttpError("source materialization camera epoch is incomplete")
camera_epoch = camera_inits[0].camera_epoch
camera_root = _secure_source_subdirectory(root, root / "camera")
camera_epoch_root = _secure_source_subdirectory(
@@ -1318,10 +1259,7 @@ def _prepare_source_destination_layout(
camera_epoch_root / "segments",
)
spatial = None
if any(
member.kind == "spatial-replay" and not member.primary
for member in members
):
if any(member.kind == "spatial-replay" and not member.primary for member in members):
spatial = _secure_source_subdirectory(root, root / "spatial")
return _SourceDestinationLayout(
root=root,
@@ -1335,9 +1273,7 @@ def _prepare_source_destination_layout(
def _secure_source_subdirectory(root: Path, candidate: Path) -> Path:
resolved = _secure_directory(candidate)
if not resolved.is_relative_to(root):
raise ObservatoryWorkerHttpError(
"source materialization destination escapes its root"
)
raise ObservatoryWorkerHttpError("source materialization destination escapes its root")
return resolved
@@ -1409,8 +1345,7 @@ def _upload_members(
or len({member.role for member in members}) != len(members)
or tuple(member.member_id for member in members)
!= tuple(sorted(member.member_id for member in members))
or sum(member.byte_length for member in members)
> 256 * 1024 * 1024 * 1024
or sum(member.byte_length for member in members) > 256 * 1024 * 1024 * 1024
or document.get("complete") != all(member.uploaded for member in members)
):
raise ObservatoryWorkerHttpError("result upload member inventory is invalid")
@@ -1489,9 +1424,7 @@ def _write_local_exact(path: Path, payload: bytes) -> None:
os.chmod(path, 0o400, follow_symlinks=False)
except FileExistsError:
if _read_local_file(path, max(1, len(payload))) != payload:
raise ObservatoryWorkerHttpError(
"Worker local manifest identity collided"
) from None
raise ObservatoryWorkerHttpError("Worker local manifest identity collided") from None
finally:
with suppress(FileNotFoundError):
temporary.unlink()
@@ -1512,9 +1445,7 @@ def _publish_local_file(
os.chmod(destination, 0o400, follow_symlinks=False)
except FileExistsError:
if not _matches_file(destination, sha256, byte_length):
raise ObservatoryWorkerHttpError(
"Worker local member publication collided"
) from None
raise ObservatoryWorkerHttpError("Worker local member publication collided") from None
def _matches_file(path: Path, sha256: str, byte_length: int) -> bool:
@@ -1530,11 +1461,7 @@ def _read_local_file(path: Path, maximum_bytes: int) -> bytes:
try:
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
before = os.fstat(descriptor)
if (
not stat.S_ISREG(before.st_mode)
or before.st_size < 0
or before.st_size > maximum_bytes
):
if not stat.S_ISREG(before.st_mode) or before.st_size < 0 or before.st_size > maximum_bytes:
raise ObservatoryWorkerHttpError("Worker local file is outside bounds")
payload = bytearray()
while len(payload) < before.st_size:
@@ -1562,11 +1489,7 @@ def _hash_local_regular_file(path: Path, maximum_bytes: int) -> tuple[str, int]:
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
before = os.fstat(descriptor)
if (
not stat.S_ISREG(before.st_mode)
or before.st_size < 0
or before.st_size > maximum_bytes
):
if not stat.S_ISREG(before.st_mode) or before.st_size < 0 or before.st_size > maximum_bytes:
raise ObservatoryWorkerHttpError("Worker local file is outside bounds")
digest = hashlib.sha256()
byte_length = 0
+14 -4
View File
@@ -52,6 +52,7 @@ from k1link.observatory.worker_http_transport import (
OBSERVATORY_WORKER_BASE_URL_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL"
OBSERVATORY_WORKER_TOKEN_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE"
OBSERVATORY_WORKER_WORK_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"
OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_SOURCE_CACHE_ROOT"
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS"
OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS_ENV: Final = (
"MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS"
@@ -140,11 +141,14 @@ class ObservatoryWorkerServiceConfiguration:
idle_poll_seconds: float = DEFAULT_OBSERVATORY_WORKER_IDLE_POLL_SECONDS
transport_backoff_seconds: float = DEFAULT_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS
max_consecutive_transport_failures: int = DEFAULT_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES
source_cache_root: Path | None = None
def __post_init__(self) -> None:
_validate_worker_base_url(self.base_url)
_absolute_path(self.bearer_token_file, "Worker bearer token file")
_absolute_path(self.work_root, "Worker work root")
if self.source_cache_root is not None:
_absolute_path(self.source_cache_root, "Worker source cache root")
if isinstance(self.idle_poll_seconds, bool) or not (
0.05 <= self.idle_poll_seconds <= 300.0
):
@@ -175,6 +179,11 @@ class ObservatoryWorkerServiceConfiguration:
),
bearer_token_file=token_file,
work_root=work_root,
source_cache_root=(
_required_path(values, OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV)
if values.get(OBSERVATORY_WORKER_SOURCE_CACHE_ROOT_ENV)
else None
),
idle_poll_seconds=_environment_float(
values,
OBSERVATORY_WORKER_IDLE_POLL_SECONDS_ENV,
@@ -265,6 +274,7 @@ def compose_installed_observatory_worker_service(
bearer_token=bearer_token,
work_root=configuration.work_root,
transport=http_transport,
source_cache_root=configuration.source_cache_root,
)
finally:
# The immutable string remains owned by the gateway headers for the
@@ -302,6 +312,7 @@ def compose_installed_observatory_worker_service_from_builders(
bearer_token=bearer_token,
work_root=configuration.work_root,
transport=http_transport,
source_cache_root=configuration.source_cache_root,
)
executors = build_ready_executor_registry(
definitions=definitions,
@@ -343,6 +354,7 @@ def compose_installed_observatory_worker_service_from_packages(
bearer_token=bearer_token,
work_root=configuration.work_root,
transport=http_transport,
source_cache_root=configuration.source_cache_root,
)
executors = build_ready_executor_registry_from_packages(
definitions=definitions,
@@ -381,9 +393,7 @@ def build_ready_executor_registry_from_packages(
_absolute_path(work_root, "Worker package build work root")
ready = definitions.ready_recorded_definitions()
ready_keys = {(item.setup_id, item.definition_sha256) for item in ready}
package_keys = {
(package.setup_id, package.definition_sha256) for package in packages.packages
}
package_keys = {(package.setup_id, package.definition_sha256) for package in packages.packages}
if not package_keys.issubset(ready_keys):
raise ObservatoryWorkerServiceError(
"installed LAB packages must bind only ready RunDefinitions"
@@ -420,7 +430,7 @@ def build_ready_executor_registry_from_packages(
if built.identity != package.executor_identity:
raise ObservatoryWorkerServiceError(
"generic package factory returned another executor identity"
)
)
registrations.append(built)
return ObservatoryWorkerExecutorRegistry(tuple(registrations))
@@ -0,0 +1,170 @@
"""Worker-local reuse of immutable source bytes, never of claim authority.
Each job must still obtain and validate its own current source manifest. Only
exact digest/length matches may populate its fixed, generation-scoped layout.
Files use read-only hardlinks on the same filesystem. An explicitly shared cache
on another filesystem uses bounded verified disk copies, never whole-file RAM.
An absent or damaged cache object is a miss; it is never overwritten or deleted.
"""
from __future__ import annotations
import errno
import hashlib
import os
import re
import secrets
import shutil
import stat
from contextlib import suppress
from pathlib import Path
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
_CHUNK_BYTES = 1024 * 1024
_FREE_RESERVE_BYTES = 2 * 1024 * 1024 * 1024
class WorkerSourceCacheError(RuntimeError):
"""A caller selected an unsafe cache root, identity or destination."""
class WorkerSourceCache:
def __init__(self, root: Path) -> None:
candidate = root.expanduser().absolute()
candidate.mkdir(mode=0o700, parents=True, exist_ok=True)
if candidate.is_symlink() or not candidate.is_dir():
raise WorkerSourceCacheError("Worker source cache root is unsafe")
self.root = candidate.resolve(strict=True)
def restore(self, destination: Path, *, sha256: str, byte_length: int) -> bool:
"""Verify cached bytes before linking them into an empty job-owned path."""
cached = self._path(sha256, byte_length)
if not cached.exists() or cached.is_symlink():
return False
if destination.exists() or destination.is_symlink():
raise WorkerSourceCacheError("Worker source cache destination is not empty")
return _publish_exact(cached, destination, sha256, byte_length)
def retain(self, source: Path, *, sha256: str, byte_length: int) -> bool:
"""Retain a fully downloaded/extracted file without copying its contents.
Corrupt existing cache files are preserved for diagnosis, not used and
not replaced. The current run can use its freshly downloaded exact file.
"""
cached = self._path(sha256, byte_length)
if cached.exists() or cached.is_symlink():
# Not consumed by this call. Restore will validate it when needed;
# do not add another whole-file read to an already-ready job.
return False
return _publish_exact(source, cached, sha256, byte_length)
def _path(self, sha256: str, byte_length: int) -> Path:
if (
not isinstance(sha256, str)
or _SHA256.fullmatch(sha256) is None
or isinstance(byte_length, bool)
or not isinstance(byte_length, int)
or byte_length < 0
):
raise WorkerSourceCacheError("Worker source cache identity is invalid")
if self.root.is_symlink() or not self.root.is_dir():
raise WorkerSourceCacheError("Worker source cache root changed")
return self.root / sha256
def _publish_exact(source: Path, destination: Path, sha256: str, byte_length: int) -> bool:
"""Hash actual staged bytes before publication; never replace a path."""
temporary = destination.with_name(f".source-cache-{secrets.token_hex(16)}")
try:
try:
os.link(source, temporary, follow_symlinks=False)
except OSError as exc:
if exc.errno != errno.EXDEV:
return False
# Different agent work mounts may share one CAS mount. Copy only
# in bounded chunks and leave a cache miss when space is low.
if shutil.disk_usage(destination.parent).free < byte_length + _FREE_RESERVE_BYTES:
return False
if not _copy_exact(source, temporary, sha256, byte_length):
return False
else:
if not _matches(temporary, sha256, byte_length):
return False
os.chmod(temporary, 0o400, follow_symlinks=False)
try:
os.link(temporary, destination, follow_symlinks=False)
except FileExistsError:
return _matches(destination, sha256, byte_length)
return True
except OSError:
# Cache publication is optional; a fully downloaded job remains valid.
return False
finally:
with suppress(FileNotFoundError):
temporary.unlink() # Only this call's temporary link/copy.
def _matches(path: Path, sha256: str, byte_length: int) -> bool:
descriptor = -1
try:
if not stat.S_ISREG(path.lstat().st_mode):
return False
descriptor = os.open(
path,
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0),
)
before = os.fstat(descriptor)
if not stat.S_ISREG(before.st_mode) or before.st_size != byte_length:
return False
digest = hashlib.sha256()
remaining = byte_length
while remaining:
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
if not chunk:
return False
digest.update(chunk)
remaining -= len(chunk)
after = os.fstat(descriptor)
named = path.lstat()
return (
stat.S_ISREG(named.st_mode)
and _stamp(before) == _stamp(after) == _stamp(named)
and digest.hexdigest() == sha256
)
except OSError:
return False
finally:
if descriptor >= 0:
os.close(descriptor)
def _stamp(value: os.stat_result) -> tuple[int, int, int, int, int]:
return value.st_dev, value.st_ino, value.st_size, value.st_mtime_ns, value.st_ctime_ns
def _copy_exact(source: Path, temporary: Path, sha256: str, byte_length: int) -> bool:
if not stat.S_ISREG(source.lstat().st_mode):
return False
descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
try:
before = os.fstat(descriptor)
if not stat.S_ISREG(before.st_mode) or before.st_size != byte_length:
return False
digest = hashlib.sha256()
remaining = byte_length
with temporary.open("xb") as target:
while remaining:
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
if not chunk:
return False
digest.update(chunk)
target.write(chunk)
remaining -= len(chunk)
target.flush()
os.fsync(target.fileno())
valid = _stamp(before) == _stamp(os.fstat(descriptor)) and digest.hexdigest() == sha256
if not valid:
temporary.unlink()
return valid
finally:
os.close(descriptor)
+11 -16
View File
@@ -19,9 +19,7 @@ from .models import ObservationSessionCandidate
EQUIPMENT_REGISTRY_SCHEMA: Final = "missioncore.equipment-model-registry/v1"
EQUIPMENT_MODEL_SCHEMA: Final = "missioncore.equipment-model/v1"
CAPTURE_PROFILE_REGISTRY_SCHEMA: Final = (
"missioncore.recorded-capture-profile-registry/v1"
)
CAPTURE_PROFILE_REGISTRY_SCHEMA: Final = "missioncore.recorded-capture-profile-registry/v1"
CAPTURE_PROFILE_SCHEMA: Final = "missioncore.recorded-capture-profile/v1"
CAPTURE_ATTESTATION_SCHEMA: Final = "missioncore.recorded-capture-attestation/v1"
@@ -119,9 +117,7 @@ class EquipmentCaptureRegistry:
@classmethod
def from_repository(cls, repository_root: Path) -> EquipmentCaptureRegistry | None:
equipment_path = repository_root / "config" / "observatory-equipment-models.json"
capture_path = (
repository_root / "config" / "observatory-recorded-capture-profiles.json"
)
capture_path = repository_root / "config" / "observatory-recorded-capture-profiles.json"
if not equipment_path.exists() and not capture_path.exists():
return None
if not equipment_path.is_file() or not capture_path.is_file():
@@ -152,9 +148,7 @@ class EquipmentCaptureRegistry:
if len(by_id) != len(equipment_models):
raise EquipmentRegistryError("equipment model ids must be unique")
capture_profiles = tuple(_capture_profile(row, by_id) for row in capture_rows)
if len({item.capture_profile_id for item in capture_profiles}) != len(
capture_profiles
):
if len({item.capture_profile_id for item in capture_profiles}) != len(capture_profiles):
raise EquipmentRegistryError("capture profile ids must be unique")
return cls(
equipment_models=equipment_models,
@@ -227,26 +221,28 @@ class EquipmentCaptureRegistry:
row = profile.document
camera = row["camera_media"]
calibration = row["calibration"]
modalities = row["modalities"]
semantic_channels = row["semantic_channels"]
assert isinstance(camera, dict)
assert isinstance(calibration, dict)
assert isinstance(modalities, list)
assert isinstance(semantic_channels, list)
required_modalities = requirements.get("required_modalities")
if (
row["plugin_id"] == requirements.get("plugin_id")
and row["archive_id"] == requirements.get("archive_id")
and row["modalities"] == requirements.get("required_modalities")
and isinstance(required_modalities, list)
and set(required_modalities).issubset(modalities)
and camera["source_id"] == requirements.get("camera_source_id")
and "camera.video.recorded" in semantic_channels
and requirements.get("camera_semantic_channel_id")
== "camera.video.recorded"
and requirements.get("camera_semantic_channel_id") == "camera.video.recorded"
and camera["media_type"] == requirements.get("recorded_media_type")
and camera["initialization_sha256"]
== requirements.get("recorded_media_init_sha256")
and camera["width"] == requirements.get("camera_width")
and camera["height"] == requirements.get("camera_height")
and calibration["slot_id"] == requirements.get("calibration_slot")
and calibration["sha256"]
== requirements.get("calibration_identity_sha256")
and calibration["sha256"] == requirements.get("calibration_identity_sha256")
and requirements.get("exactly_one_media_epoch") is True
and requirements.get("seekable") is True
):
@@ -336,8 +332,7 @@ def _capture_profile(
or not channels
or channels != sorted(channels)
or any(
not isinstance(item, str) or _IDENTIFIER.fullmatch(item) is None
for item in channels
not isinstance(item, str) or _IDENTIFIER.fullmatch(item) is None for item in channels
)
):
raise EquipmentRegistryError("capture semantic channels are invalid")
+96 -70
View File
@@ -2,17 +2,20 @@
from __future__ import annotations
from collections import OrderedDict
from collections.abc import Callable
from contextlib import suppress
from threading import Lock
from typing import Any, Literal
from uuid import UUID
from uuid import UUID, uuid5
import rerun as rr
import rerun_bindings as bindings
from rerun import blueprint as rrb
from k1link.viewer.recorded_blueprint_lifecycle import (
BlueprintSessionReleased,
RecordedBlueprintSessions,
)
from k1link.viewer.rerun_bridge import RerunSceneSettings
APPLICATION_ID = "nodedc_mission_core_recorded"
@@ -48,8 +51,9 @@ class _RecordedBlueprintStream:
"""Keep one bounded SDK blueprint source for each browser viewport.
Upstream 0.36.3 activates a clone, not this source store. Explicit refresh
makes layer changes visible but cannot retain the clone's operator eye.
It is opt-in for portable replay pending native camera-state support.
makes layer changes visible. Ordinary layer updates deliberately omit eye
components so the active clone retains the operator's camera. Only an
explicit 3D/plan/follow transition may write a new spatial eye preset.
"""
def __init__(
@@ -57,8 +61,10 @@ class _RecordedBlueprintStream:
application_id: str,
recording_id: str,
*,
blueprint_session_id: str,
view_reset_generation: Literal[0, 1],
) -> None:
self.blueprint_session_id = blueprint_session_id
self.view_reset_generation = view_reset_generation
self._lock = Lock()
self._sequence = 0
@@ -81,7 +87,7 @@ class _RecordedBlueprintStream:
def render(
self,
blueprint_factory: Callable[[bool, bool], rrb.Blueprint],
blueprint_factory: Callable[[bool, bool, str], rrb.Blueprint],
*,
follow_trajectory: bool,
plan_view: bool,
@@ -94,10 +100,20 @@ class _RecordedBlueprintStream:
update_eye_controls = self._eye_contract != eye_contract
# Initial admission/reset uses native framing. Explicit presets
# apply to mode transitions, after the viewer has a source cursor.
blueprint = blueprint_factory(update_eye_controls, self._eye_contract is not None)
# Keep one view identity for the lifetime of this browser owner.
# Replacing the UUID on every layer toggle rebuilt both native
# viewports, delayed a simple visibility change, and discarded the
# operator's active interaction state. The explicit reset
# generation already owns the one intentional identity change.
view_instance_token = self.blueprint_session_id
blueprint = blueprint_factory(
update_eye_controls,
self._eye_contract is not None,
view_instance_token,
)
# Appending rows alone does not refresh upstream's active clone.
# Keep legacy admission unchanged; portable replay opts into
# working layer updates with an explicitly documented eye reset.
# Reactivation makes layer changes visible; eye components remain
# absent unless the explicit view-mode contract changed above.
make_active = self._sequence == 0 or reactivate_updates
self._blueprint_recording.set_time(
"blueprint",
@@ -127,40 +143,7 @@ class _RecordedBlueprintStream:
self._transport_recording.disconnect()
_MAX_RECORDED_BLUEPRINT_STREAMS = 32
_recorded_blueprint_streams_lock = Lock()
_recorded_blueprint_streams: OrderedDict[tuple[str, str, str], _RecordedBlueprintStream] = (
OrderedDict()
)
def _stable_recorded_blueprint_stream(
*,
application_id: str,
recording_id: str,
blueprint_session_id: str,
view_reset_generation: Literal[0, 1],
) -> _RecordedBlueprintStream:
key = (application_id, recording_id, blueprint_session_id)
with _recorded_blueprint_streams_lock:
stream = _recorded_blueprint_streams.get(key)
if stream is not None and stream.view_reset_generation != view_reset_generation:
stream.close()
del _recorded_blueprint_streams[key]
stream = None
if stream is None:
stream = _RecordedBlueprintStream(
application_id,
recording_id,
view_reset_generation=view_reset_generation,
)
_recorded_blueprint_streams[key] = stream
else:
_recorded_blueprint_streams.move_to_end(key)
while len(_recorded_blueprint_streams) > _MAX_RECORDED_BLUEPRINT_STREAMS:
_, stale = _recorded_blueprint_streams.popitem(last=False)
stale.close()
return stream
recorded_blueprint_sessions = RecordedBlueprintSessions[_RecordedBlueprintStream]()
def recorded_blueprint(
@@ -170,15 +153,21 @@ def recorded_blueprint(
active_view: RecordedView = "spatial",
view_reset_generation: Literal[0, 1] = 0,
unified_perception: bool = False,
unified_camera_share: float = 0.46,
semantic_layer: Literal["city", "vegetation"] | None = None,
plan_view: bool = False,
show_detections_2d: bool = False,
show_camera_image: bool = True,
show_segmentation: bool = False,
show_cuboids_3d: bool = False,
show_costmap: bool = False,
follow_trajectory: bool = False,
update_eye_controls: bool = True,
explicit_spatial_preset: bool = False,
eye_position: tuple[float, float, float] | None = None,
eye_look_target: tuple[float, float, float] | None = None,
eye_up: tuple[float, float, float] | None = None,
view_instance_token: str | None = None,
) -> rrb.Blueprint:
accumulation = max(0.0, settings.accumulation_seconds)
time_ranges: list[rr.VisibleTimeRange] | None = None
@@ -218,6 +207,14 @@ def recorded_blueprint(
)
spatial_eye_controls = (
rrb.EyeControls3D.from_fields(
kind=rrb.Eye3DKind.Orbital,
position=eye_position,
look_target=eye_look_target,
eye_up=eye_up,
tracking_entity="/world/sensor_pose" if follow_trajectory else "",
)
if eye_position is not None and eye_look_target is not None and eye_up is not None
else rrb.EyeControls3D.from_fields(
kind=rrb.Eye3DKind.Orbital,
position=[0.0, 0.0, 30.0],
look_target=[0.0, 0.0, 0.0],
@@ -267,15 +264,21 @@ def recorded_blueprint(
# alone does not preserve edits in upstream's activated blueprint clone.
eye_controls=spatial_eye_controls,
)
spatial_view.id = (
RECORDED_SPATIAL_RESET_VIEW_ID if view_reset_generation else RECORDED_SPATIAL_VIEW_ID
def instance_id(primary: UUID, reset: UUID) -> UUID:
base = reset if view_reset_generation else primary
return uuid5(base, view_instance_token) if view_instance_token is not None else base
spatial_view.id = instance_id(
RECORDED_SPATIAL_VIEW_ID,
RECORDED_SPATIAL_RESET_VIEW_ID,
)
camera_view = rrb.Spatial2DView(
origin="/perception/camera",
name="Оригинальное видео · слои AI",
background=[7, 8, 10, 255],
overrides={
"/perception/camera/image": rrb.EntityBehavior(visible=True),
"/perception/camera/image": rrb.EntityBehavior(visible=show_camera_image),
"/perception/camera/detections": rrb.EntityBehavior(
visible=show_detections_2d,
),
@@ -290,8 +293,9 @@ def recorded_blueprint(
),
},
)
camera_view.id = (
RECORDED_CAMERA_RESET_VIEW_ID if view_reset_generation else RECORDED_CAMERA_VIEW_ID
camera_view.id = instance_id(
RECORDED_CAMERA_VIEW_ID,
RECORDED_CAMERA_RESET_VIEW_ID,
)
perception_3d_view = rrb.Spatial3DView(
# Cuboids are expressed in the same calibrated world frame as the
@@ -325,10 +329,9 @@ def recorded_blueprint(
},
eye_controls=spatial_eye_controls,
)
perception_3d_view.id = (
RECORDED_PERCEPTION_3D_RESET_VIEW_ID
if view_reset_generation
else RECORDED_PERCEPTION_3D_VIEW_ID
perception_3d_view.id = instance_id(
RECORDED_PERCEPTION_3D_VIEW_ID,
RECORDED_PERCEPTION_3D_RESET_VIEW_ID,
)
metrics_view = rrb.TimeSeriesView(
origin="/metrics/device",
@@ -357,16 +360,16 @@ def recorded_blueprint(
spatial_view.visualizer_overrides["/world/perception/boxes3d"] = [
rrb.EntityBehavior(visible=show_cuboids_3d),
]
camera_share = min(0.9, max(0.1, unified_camera_share))
root_container = rrb.Horizontal(
camera_view,
spatial_view,
column_shares=[0.46, 0.54],
column_shares=[camera_share, 1.0 - camera_share],
name="Единая сцена восприятия",
)
root_container.id = (
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID
if view_reset_generation
else RECORDED_UNIFIED_ROOT_CONTAINER_ID
root_container.id = instance_id(
RECORDED_UNIFIED_ROOT_CONTAINER_ID,
RECORDED_UNIFIED_RESET_ROOT_CONTAINER_ID,
)
else:
# Keep operator video and 3D cuboids as direct root views. Rerun's nested
@@ -384,7 +387,7 @@ def recorded_blueprint(
# replace it from a later blueprint message. Give every operator mode (and
# its explicit reset generation) a stable root identity so the requested
# child is authoritative instead of inheriting a previously visited tab.
root_container.id = {
base_root_id = {
("spatial", 0): RECORDED_ROOT_CONTAINER_ID,
("spatial", 1): RECORDED_SPATIAL_RESET_ROOT_CONTAINER_ID,
("perception", 0): RECORDED_PERCEPTION_ROOT_CONTAINER_ID,
@@ -394,6 +397,11 @@ def recorded_blueprint(
("metrics", 0): RECORDED_METRICS_ROOT_CONTAINER_ID,
("metrics", 1): RECORDED_METRICS_RESET_ROOT_CONTAINER_ID,
}[(active_view, view_reset_generation)]
root_container.id = (
uuid5(base_root_id, view_instance_token)
if view_instance_token is not None
else base_root_id
)
if include_initial_playback_state:
return rrb.Blueprint(
@@ -424,19 +432,26 @@ def recorded_blueprint_rrd(
active_view: RecordedView = "spatial",
view_reset_generation: Literal[0, 1] = 0,
unified_perception: bool = False,
unified_camera_share: float = 0.46,
semantic_layer: Literal["city", "vegetation"] | None = None,
plan_view: bool = False,
show_detections_2d: bool = False,
show_camera_image: bool = True,
show_segmentation: bool = False,
show_cuboids_3d: bool = False,
show_costmap: bool = False,
follow_trajectory: bool = False,
reactivate_updates: bool = False,
eye_position: tuple[float, float, float] | None = None,
eye_look_target: tuple[float, float, float] | None = None,
eye_up: tuple[float, float, float] | None = None,
) -> bytes:
"""Serialize a bounded active blueprint update without recorded data."""
def build_blueprint(
update_eye_controls: bool, use_spatial_preset: bool = False
update_eye_controls: bool,
use_spatial_preset: bool = False,
view_instance_token: str | None = None,
) -> rrb.Blueprint:
return recorded_blueprint(
settings,
@@ -444,32 +459,43 @@ def recorded_blueprint_rrd(
active_view=active_view,
view_reset_generation=view_reset_generation,
unified_perception=unified_perception,
unified_camera_share=unified_camera_share,
semantic_layer=semantic_layer,
plan_view=plan_view,
show_detections_2d=show_detections_2d,
show_camera_image=show_camera_image,
show_segmentation=show_segmentation,
show_cuboids_3d=show_cuboids_3d,
show_costmap=show_costmap,
follow_trajectory=follow_trajectory,
update_eye_controls=update_eye_controls,
explicit_spatial_preset=reactivate_updates and use_spatial_preset,
explicit_spatial_preset=use_spatial_preset and update_eye_controls,
eye_position=eye_position,
eye_look_target=eye_look_target,
eye_up=eye_up,
view_instance_token=view_instance_token,
)
payload: bytes | None
if blueprint_session_id is not None:
try:
payload = _stable_recorded_blueprint_stream(
application_id=application_id,
recording_id=recording_id,
blueprint_session_id=blueprint_session_id,
view_reset_generation=view_reset_generation,
).render(
build_blueprint,
follow_trajectory=follow_trajectory,
plan_view=plan_view,
reactivate_updates=reactivate_updates,
payload = recorded_blueprint_sessions.use(
(application_id, recording_id, blueprint_session_id),
view_reset_generation,
lambda: _RecordedBlueprintStream(
application_id,
recording_id,
blueprint_session_id=blueprint_session_id,
view_reset_generation=view_reset_generation,
),
lambda stream: stream.render(
build_blueprint,
follow_trajectory=follow_trajectory,
plan_view=plan_view,
reactivate_updates=reactivate_updates,
),
)
except RecordedBlueprintError:
except (RecordedBlueprintError, BlueprintSessionReleased):
raise
except Exception as exc:
raise RecordedBlueprintError("failed to serialize stable recorded blueprint") from exc
@@ -0,0 +1,129 @@
"""Ephemeral viewport resources: renew while mounted, release at termination."""
from __future__ import annotations
import time
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from threading import RLock
from typing import Protocol
BlueprintKey = tuple[str, str, str]
class Closable(Protocol):
def close(self) -> None: ...
class BlueprintSessionReleased(RuntimeError):
"""A late update cannot resurrect an explicitly released viewport."""
@dataclass
class _Entry[Resource: Closable]:
resource: Resource
generation: int
last_used: float
class RecordedBlueprintSessions[Resource: Closable]:
"""No TTL applies while a render is running; idle owners renew separately.
The small registry lock also serializes release with render. Thus the
release acknowledgement means native resources have actually been closed,
and a previously queued update is fenced by a lightweight tombstone.
"""
def __init__(
self,
*,
ttl_seconds: float = 300.0,
max_entries: int = 32,
clock: Callable[[], float] = time.monotonic,
) -> None:
self._ttl = ttl_seconds
self._max_entries = max_entries
self._clock = clock
self._lock = RLock()
self._entries: OrderedDict[BlueprintKey, _Entry[Resource]] = OrderedDict()
self._released: OrderedDict[BlueprintKey, float] = OrderedDict()
def _expire(self, now: float) -> None:
for key, entry in list(self._entries.items()):
if now - entry.last_used >= self._ttl:
del self._entries[key]
entry.resource.close()
for key, expires in list(self._released.items()):
if expires <= now:
del self._released[key]
def use[Result](
self,
key: BlueprintKey,
generation: int,
create: Callable[[], Resource],
render: Callable[[Resource], Result],
) -> Result:
with self._lock:
now = self._clock()
self._expire(now)
if key in self._released:
raise BlueprintSessionReleased("recorded viewport has been released")
entry = self._entries.get(key)
if entry is not None and entry.generation != generation:
del self._entries[key]
entry.resource.close()
entry = None
if entry is None:
entry = _Entry(create(), generation, now)
self._entries[key] = entry
self._entries.move_to_end(key)
while len(self._entries) > self._max_entries:
_, stale = self._entries.popitem(last=False)
stale.resource.close()
try:
return render(entry.resource)
except BaseException:
# A failed operation must not retain its partial native store.
del self._entries[key]
entry.resource.close()
raise
finally:
entry.last_used = self._clock()
def renew(self, key: BlueprintKey) -> bool:
with self._lock:
now = self._clock()
self._expire(now)
entry = self._entries.get(key)
if entry is None:
return False
entry.last_used = now
self._entries.move_to_end(key)
return True
def release(self, key: BlueprintKey) -> None:
with self._lock:
now = self._clock()
self._expire(now)
entry = self._entries.pop(key, None)
self._released[key] = now + self._ttl
self._released.move_to_end(key)
# Tombstones contain identities only, never SDK resources or data.
while len(self._released) > 4096:
self._released.popitem(last=False)
if entry is not None:
entry.resource.close()
def expire(self) -> None:
with self._lock:
self._expire(self._clock())
def close(self) -> None:
with self._lock:
entries = list(self._entries.values())
self._entries.clear()
self._released.clear()
for entry in entries:
entry.resource.close()
+187
View File
@@ -0,0 +1,187 @@
"""Scene-derived orbital camera bounds for recorded Rerun views."""
from __future__ import annotations
import math
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
import numpy as np
import pyarrow as pa
import pyarrow.compute as pc
import rerun_bindings as bindings
SESSION_TIMELINE = "session_time"
MIN_ORBIT_DISTANCE = 0.02
MAX_ORBITAL_ZOOM_OUT_FACTOR = 5.0
@dataclass(frozen=True)
class _SpatialBoundsSeries:
times_ns: np.ndarray
lower: np.ndarray
upper: np.ndarray
@lru_cache(maxsize=8)
def _recorded_spatial_bounds_index(
path_text: str,
byte_length: int,
modified_ns: int,
) -> dict[str, _SpatialBoundsSeries]:
"""Build a small temporal bounds index for one immutable RRD generation.
Layer and follow buttons only change a blueprint. Re-decoding the complete
source RRD for every click made those controls wait on archive I/O. The
cache key includes the file generation fingerprint; cached values contain
only timestamps and six floats per sample, never point-cloud payloads.
"""
del byte_length, modified_ns # Generation identity is carried by the cache key.
reader = bindings.RrdReaderInternal(path_text)
recording = next(
(entry for entry in reader.store_entries() if entry.kind == "recording"),
None,
)
if recording is None:
raise ValueError("recorded camera source has no recording store")
rows: dict[str, list[tuple[int, np.ndarray, np.ndarray]]] = {
"/world/points": [],
"/world/trajectory": [],
}
for chunk in reader.stream(recording):
entity_path = str(chunk.entity_path)
if entity_path not in rows:
continue
batch = chunk.to_record_batch()
names = batch.column_names
if SESSION_TIMELINE not in names:
continue
component = (
"Points3D:positions" if entity_path == "/world/points" else "LineStrips3D:strips"
)
if component not in names:
continue
times = np.asarray(
batch.column(names.index(SESSION_TIMELINE)).cast(pa.int64()),
dtype=np.int64,
)
spatial = batch.column(names.index(component))
for index, timestamp in enumerate(times):
values = _spatial_values(
spatial.slice(index, 1),
nested=entity_path == "/world/trajectory",
)
if not values.size:
continue
finite = values[np.isfinite(values).all(axis=1)]
if not finite.size:
continue
rows[entity_path].append(
(
int(timestamp),
np.min(finite, axis=0),
np.max(finite, axis=0),
)
)
result: dict[str, _SpatialBoundsSeries] = {}
for entity_path, samples in rows.items():
if not samples:
continue
samples.sort(key=lambda sample: sample[0])
result[entity_path] = _SpatialBoundsSeries(
times_ns=np.asarray([sample[0] for sample in samples], dtype=np.int64),
lower=np.asarray([sample[1] for sample in samples], dtype=np.float32),
upper=np.asarray([sample[2] for sample in samples], dtype=np.float32),
)
return result
def recorded_orbital_radius_limit(
recording_path: Path,
*,
current_time_ns: int,
accumulation_seconds: float,
show_points: bool,
show_trajectory: bool,
) -> float | None:
"""Match Rerun 0.36.3's current scene-diagonal zoom-out limit.
The LAB 3D view roots its native mapping data at ``/world``. Points and
trajectory are the source entities whose visible-time query changes with
the operator's accumulation setting. Derived layers are deliberately not
folded into this source contract: they use the same calibrated world frame
and do not own navigation.
"""
if current_time_ns < 0 or not math.isfinite(accumulation_seconds) or accumulation_seconds < 0:
raise ValueError("invalid recorded camera query")
wanted = {
*(("/world/points",) if show_points else ()),
*(("/world/trajectory",) if show_trajectory else ()),
}
if not wanted:
return None
path = recording_path.expanduser().absolute()
if path.is_symlink() or not path.is_file():
raise ValueError("recorded camera source is unavailable")
stat = path.stat()
series_by_entity = _recorded_spatial_bounds_index(
str(path),
stat.st_size,
stat.st_mtime_ns,
)
lower_ns = current_time_ns - round(accumulation_seconds * 1_000_000_000)
lower = np.array([np.inf, np.inf, np.inf], dtype=np.float32)
upper = np.array([-np.inf, -np.inf, -np.inf], dtype=np.float32)
found = False
for entity_path in wanted:
series = series_by_entity.get(entity_path)
if series is None:
continue
if accumulation_seconds > 0:
selected = (series.times_ns >= lower_ns) & (series.times_ns <= current_time_ns)
else:
eligible_end = int(np.searchsorted(series.times_ns, current_time_ns, side="right"))
if eligible_end == 0:
continue
latest_time = series.times_ns[eligible_end - 1]
selected = series.times_ns == latest_time
if not selected.any():
continue
lower = np.minimum(lower, np.min(series.lower[selected], axis=0))
upper = np.maximum(upper, np.max(series.upper[selected], axis=0))
found = True
if not found:
return None
# macaw::BoundingBox and Rerun's eye controller operate in f32.
diagonal = np.float32(np.linalg.norm((upper - lower).astype(np.float32)))
if not np.isfinite(diagonal) or diagonal <= 0:
return None
return float(
max(
np.float32(MIN_ORBIT_DISTANCE),
np.float32(diagonal * np.float32(MAX_ORBITAL_ZOOM_OUT_FACTOR)),
)
)
def _spatial_values(column: pa.Array, *, nested: bool) -> np.ndarray:
flattened = pc.list_flatten(column)
if nested:
flattened = pc.list_flatten(flattened)
if not pa.types.is_fixed_size_list(flattened.type) or flattened.type.list_size != 3:
raise ValueError("recorded spatial component is not a 3D vector")
# Flatten through Arrow rather than reading ``FixedSizeListArray.values``:
# the latter exposes the complete backing buffer and ignores a sliced
# array's logical offset.
values = pc.list_flatten(flattened).to_numpy(zero_copy_only=False)
if values.size == 0:
return np.empty((0, 3), dtype=np.float32)
return np.asarray(values, dtype=np.float32).reshape((-1, 3))
+108 -64
View File
@@ -42,11 +42,19 @@ from k1link.observatory import (
ObservatoryRunPreparationLedger,
load_observatory_run_preparation_ledger,
)
from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
from k1link.observatory.domain_ontology import (
ObservatoryDomainOntology,
ObservatoryOntologyError,
)
from k1link.observatory.lab_view_profiles import LabViewProfileError, LabViewProfileStore
from k1link.observatory.m49_queue_binding import (
M49QueueBindingConfig,
M49QueueBindingError,
M49RecordedQueueBindingService,
)
from k1link.observatory.modular_composition import CompositionError, ModuleRegistry
from k1link.observatory.modular_composition_store import ModularCompositionStore
from k1link.observatory.portable_publication_reconciler import (
PortablePublicationReconciler,
)
@@ -196,6 +204,7 @@ from k1link.web.map_api import (
build_map_router,
)
from k1link.web.map_view_api import build_map_view_router
from k1link.web.modular_observatory_api import build_modular_observatory_router
from k1link.web.observatory_api import build_observatory_router
from k1link.web.observatory_worker_api import (
ObservatoryWorkerAuthentication,
@@ -288,6 +297,37 @@ plugin_environment = load_installed_device_plugins(REPOSITORY_ROOT)
plugin_catalog: DevicePluginCatalog = plugin_environment.catalog
plugin_dispatcher: DevicePluginDispatcher = plugin_environment.dispatcher
session_store = SessionStore(REPOSITORY_ROOT)
OBSERVATORY_AI_COMPOSITIONS: ModularCompositionStore | None
OBSERVATORY_AI_COMPOSITION_RUNS: CompositionRunStore | None
OBSERVATORY_DOMAIN_ONTOLOGY: ObservatoryDomainOntology | None
OBSERVATORY_LAB_VIEW_PROFILES: LabViewProfileStore | None
try:
OBSERVATORY_DOMAIN_ONTOLOGY = ObservatoryDomainOntology.from_file(
REPOSITORY_ROOT / "config" / "observatory-domain-ontology.json"
)
OBSERVATORY_AI_COMPOSITIONS = ModularCompositionStore(
session_store.data_dir / "observatory-ai-compositions",
ModuleRegistry.from_file(REPOSITORY_ROOT / "config" / "observatory-ai-modules.json"),
)
OBSERVATORY_AI_COMPOSITION_RUNS = CompositionRunStore(
session_store.data_dir / "observatory-ai-composition-runs"
)
OBSERVATORY_LAB_VIEW_PROFILES = LabViewProfileStore(
session_store.data_dir / "observatory-lab-view-profiles"
)
except (
CompositionError,
CompositionRunError,
LabViewProfileError,
ObservatoryOntologyError,
OSError,
ValueError,
):
# A modular catalog failure cannot disable recordings, existing LABs or Legacy.
OBSERVATORY_AI_COMPOSITIONS = None
OBSERVATORY_AI_COMPOSITION_RUNS = None
OBSERVATORY_DOMAIN_ONTOLOGY = None
OBSERVATORY_LAB_VIEW_PROFILES = None
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY: PortableRunDefinitionRegistry | None
OBSERVATORY_PORTABLE_DEFINITION_REGISTRY_ERROR: str | None
@@ -322,9 +362,7 @@ def _resolve_observatory_calculation_profile(
summary: SessionSummary,
) -> dict[str, object] | None:
if OBSERVATORY_LABORATORY_SETUP_REGISTRY is not None:
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(
summary
)
legacy = OBSERVATORY_LABORATORY_SETUP_REGISTRY.observatory_calculation_profile(summary)
if legacy is not None:
return legacy
if (
@@ -437,47 +475,30 @@ try:
or "portable definition registry is unavailable"
)
if OBSERVATORY_PORTABLE_CALCULATION_PROFILES is None:
raise PortableWorkerIntegrationError(
"portable calculation profile registry is unavailable"
)
raise PortableWorkerIntegrationError("portable calculation profile registry is unavailable")
if OBSERVATORY_PORTABLE_RESULT_VALIDATORS is None:
raise PortableWorkerIntegrationError(
"portable result validator registry is unavailable"
)
raise PortableWorkerIntegrationError("portable result validator registry is unavailable")
if OBSERVATORY_RECORDED_JOB_QUEUE is None:
raise PortableWorkerIntegrationError(
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR
or "Observatory recorded-job queue is unavailable"
OBSERVATORY_RECORDED_JOB_QUEUE_ERROR or "Observatory recorded-job queue is unavailable"
)
if session_artifact_gateway is None:
raise PortableWorkerIntegrationError(
"central artifact store is not configured"
)
raise PortableWorkerIntegrationError("central artifact store is not configured")
if session_artifact_gateway.status().central_status != "ready":
raise PortableWorkerIntegrationError(
"central artifact store is unavailable"
)
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = (
PortableWorkerStorageRoots.from_environment(
artifact_store_root=session_artifact_gateway.store.root,
)
raise PortableWorkerIntegrationError("central artifact store is unavailable")
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS = PortableWorkerStorageRoots.from_environment(
artifact_store_root=session_artifact_gateway.store.root,
)
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = (
build_portable_observatory_worker_integration(
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
artifact_store=session_artifact_gateway.store,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
source_cas_root=(
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root
),
result_staging_root=(
OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root
),
)
OBSERVATORY_PORTABLE_WORKER_INTEGRATION = build_portable_observatory_worker_integration(
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
session_store=session_store,
media_inspector=session_recorded_media_inspector,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
artifact_store=session_artifact_gateway.store,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
validators=OBSERVATORY_PORTABLE_RESULT_VALIDATORS,
source_cas_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.source_cas_root),
result_staging_root=(OBSERVATORY_PORTABLE_WORKER_STORAGE_ROOTS.result_staging_root),
)
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = None
except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
@@ -487,10 +508,7 @@ except (PortableWorkerIntegrationError, OSError, ValueError) as exc:
OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR = str(exc)
OBSERVATORY_PUBLICATION_RECONCILER = (
None
if (
OBSERVATORY_RECORDED_JOB_QUEUE is None
or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None
)
if (OBSERVATORY_RECORDED_JOB_QUEUE is None or OBSERVATORY_PORTABLE_WORKER_INTEGRATION is None)
else PortablePublicationReconciler(
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
artifact_transport=OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport,
@@ -499,12 +517,10 @@ OBSERVATORY_PUBLICATION_RECONCILER = (
)
OBSERVATORY_WORKER_API_GATE_ENABLED = OBSERVATORY_WORKER_LOCAL_ENABLED
OBSERVATORY_WORKER_CLAIM_LEASE_READY = (
OBSERVATORY_WORKER_API_GATE_ENABLED
and OBSERVATORY_RECORDED_JOB_QUEUE is not None
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_RECORDED_JOB_QUEUE is not None
)
OBSERVATORY_WORKER_VERIFIED_RESULT_PUBLISHER_READY = (
OBSERVATORY_WORKER_API_GATE_ENABLED
and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
OBSERVATORY_WORKER_API_GATE_ENABLED and OBSERVATORY_PORTABLE_WORKER_INTEGRATION is not None
)
OBSERVATORY_WORKER_DISPATCH_READY = (
OBSERVATORY_WORKER_CLAIM_LEASE_READY
@@ -527,17 +543,13 @@ else:
)
if OBSERVATORY_WORKER_AUTHENTICATION_ERROR is not None:
worker_api_errors.append(
"authentication unavailable: "
f"{OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
f"authentication unavailable: {OBSERVATORY_WORKER_AUTHENTICATION_ERROR}"
)
if OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR is not None:
worker_api_errors.append(
"integration unavailable: "
f"{OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
f"integration unavailable: {OBSERVATORY_PORTABLE_WORKER_INTEGRATION_ERROR}"
)
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(
worker_api_errors
)
OBSERVATORY_WORKER_API_ERROR = "Worker pull API is disabled; " + "; ".join(worker_api_errors)
OBSERVATORY_PORTABLE_BINDING_SERVICE: PortableRecordedQueueBindingService | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR: PortableSetupProjector | None
OBSERVATORY_PORTABLE_SETUP_PROJECTOR_ERROR: str | None
@@ -554,7 +566,8 @@ try:
and OBSERVATORY_PORTABLE_CALCULATION_PROFILES is not None
):
OBSERVATORY_PORTABLE_RESULT_CACHE = PortableResultCache(
sessions=session_store, artifacts=session_artifact_gateway.store,
sessions=session_store,
artifacts=session_artifact_gateway.store,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
calculation_profiles=OBSERVATORY_PORTABLE_CALCULATION_PROFILES,
@@ -566,7 +579,8 @@ try:
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
published_result_available=(
None if OBSERVATORY_PORTABLE_RESULT_CACHE is None
None
if OBSERVATORY_PORTABLE_RESULT_CACHE is None
else OBSERVATORY_PORTABLE_RESULT_CACHE.available
),
)
@@ -874,10 +888,19 @@ async def _portable_result_publication_reconciler() -> None:
await asyncio.sleep(15.0)
async def _recorded_blueprint_resource_reaper() -> None:
from k1link.viewer.recorded import recorded_blueprint_sessions
while True:
await asyncio.sleep(30.0)
await asyncio.to_thread(recorded_blueprint_sessions.expire)
@asynccontextmanager
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
reconciler: asyncio.Task[None] | None = None
publication_reconciler: asyncio.Task[None] | None = None
blueprint_reaper: asyncio.Task[None] | None = None
try:
configure_scanner_diagnostics(session_store.data_dir / "logs")
session_recording_preparation_manager.start()
@@ -891,11 +914,17 @@ async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
# expensive on field captures. Start it immediately in the background
# instead of holding the ASGI startup gate.
reconciler = asyncio.create_task(_recording_preparation_reconciler())
publication_reconciler = asyncio.create_task(
_portable_result_publication_reconciler()
)
publication_reconciler = asyncio.create_task(_portable_result_publication_reconciler())
blueprint_reaper = asyncio.create_task(_recorded_blueprint_resource_reaper())
yield
finally:
from k1link.viewer.recorded import recorded_blueprint_sessions
if blueprint_reaper is not None:
blueprint_reaper.cancel()
with suppress(asyncio.CancelledError):
await blueprint_reaper
await asyncio.to_thread(recorded_blueprint_sessions.close)
await map_gateway_proxy.close()
if reconciler is not None:
reconciler.cancel()
@@ -1066,7 +1095,9 @@ if session_artifact_gateway is not None and _ffmpeg is not None:
media=session_recorded_media_inspector,
recording_source=_canonical_lab_recording_source,
ffmpeg_path=_ffmpeg,
)
),
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
)
)
@@ -1128,6 +1159,23 @@ app.include_router(
),
)
)
if (
OBSERVATORY_AI_COMPOSITIONS is not None
and OBSERVATORY_AI_COMPOSITION_RUNS is not None
and OBSERVATORY_DOMAIN_ONTOLOGY is not None
):
app.include_router(
build_modular_observatory_router(
store=session_store,
compositions=OBSERVATORY_AI_COMPOSITIONS,
composition_runs=OBSERVATORY_AI_COMPOSITION_RUNS,
ontology=OBSERVATORY_DOMAIN_ONTOLOGY,
definitions=OBSERVATORY_PORTABLE_DEFINITION_REGISTRY,
binding=OBSERVATORY_PORTABLE_BINDING_SERVICE,
queue=OBSERVATORY_RECORDED_JOB_QUEUE,
view_profiles=OBSERVATORY_LAB_VIEW_PROFILES,
)
)
if OBSERVATORY_WORKER_DISPATCH_READY:
assert OBSERVATORY_RECORDED_JOB_QUEUE is not None
assert OBSERVATORY_WORKER_AUTHENTICATION is not None
@@ -1136,12 +1184,8 @@ if OBSERVATORY_WORKER_DISPATCH_READY:
build_observatory_worker_router(
OBSERVATORY_RECORDED_JOB_QUEUE,
authentication=OBSERVATORY_WORKER_AUTHENTICATION,
artifact_transport=(
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport
),
result_publisher=(
OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher
),
artifact_transport=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.artifact_transport),
result_publisher=(OBSERVATORY_PORTABLE_WORKER_INTEGRATION.result_publisher),
)
)
app.include_router(
+469
View File
@@ -0,0 +1,469 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, ConfigDict, Field
from k1link.observatory.composition_runs import (
CompositionRun,
CompositionRunError,
CompositionRunStore,
)
from k1link.observatory.domain_ontology import ObservatoryDomainOntology, ObservatoryOntologyError
from k1link.observatory.lab_view_profiles import (
PROFILE_SCHEMA as LAB_VIEW_PROFILE_SCHEMA,
)
from k1link.observatory.lab_view_profiles import (
LabSceneProfile,
LabViewProfile,
LabViewProfileError,
LabViewProfileStore,
)
from k1link.observatory.modular_composition import COMPOSITION_SCHEMA, CompositionError
from k1link.observatory.modular_composition_store import ModularCompositionStore
from k1link.observatory.portable_queue_binding import (
PortableQueueBindingError,
PortableRecordedQueueBindingService,
)
from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueDuplicateError,
ObservatoryRecordedQueueError,
)
from k1link.observatory.source_admission import PortableSourceNotPreparedError
from k1link.sessions import SessionNotFoundError, SessionStore
_EXECUTABLE_SINGLE_MODULE_SETUPS = {
"ddrnet": "ai-segmentation-ddrnet-v1",
"eomt": "ai-segmentation-eomt-v1",
"tgs": "m49-tgs-portable-v2",
"rf-detr": "ai-detection-rf-detr-v1",
"object-distance": "ai-range-object-distance-v1",
}
def _composition_error_detail(error: CompositionError) -> str:
detail = str(error)
if detail.startswith("unsupported value for "):
return (
"Параметры выбранного AI-модуля устарели. "
"Закройте окно, откройте его снова и повторите расчёт."
)
if detail == "module version is not installed":
return (
"Версия выбранного AI-модуля обновилась. "
"Закройте окно, откройте его снова и повторите расчёт."
)
if detail.startswith("select only one provider for "):
return "В одном слое можно выбрать только один AI-модуль."
if (
detail.startswith("select a module providing ")
or detail.startswith("ambiguous provider for ")
or detail == "unresolved module dependencies"
or detail == "cyclic module dependencies"
):
return "Для выбранной конфигурации не хватает обязательного связанного модуля."
if detail == "select at least one AI module":
return "Выберите хотя бы один AI-модуль."
return "Конфигурацию AI-слоя не удалось проверить. Обновите окно и повторите выбор."
class _Strict(BaseModel):
model_config = ConfigDict(extra="forbid")
class AICompositionRequest(_Strict):
schema_version: str
source_session_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
selections: list[dict[str, Any]] = Field(min_length=1, max_length=6)
idempotency_key: str = Field(
min_length=1,
max_length=160,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$",
)
class AICompositionRunRenameRequest(_Strict):
schema_version: str
display_name: str = Field(min_length=1, max_length=160)
class LabSceneProfileRequest(_Strict):
point_size: float = Field(ge=0.1, allow_inf_nan=False)
accumulation_seconds: float = Field(ge=0, allow_inf_nan=False)
color_mode: str = Field(pattern=r"^(intensity|height|distance|rgb|class)$")
palette: str = Field(pattern=r"^(turbo|viridis|plasma|grayscale)$")
show_grid: bool
show_labels: bool
show_camera_frustums: bool
class LabViewProfileRequest(_Strict):
schema_version: str
result_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$")
scene_settings: LabSceneProfileRequest
def build_modular_observatory_router(
*,
store: SessionStore,
compositions: ModularCompositionStore,
composition_runs: CompositionRunStore | None = None,
ontology: ObservatoryDomainOntology | None = None,
definitions: PortableRunDefinitionRegistry | None = None,
binding: PortableRecordedQueueBindingService | None = None,
queue: ObservatoryRecordedJobQueue | None = None,
view_profiles: LabViewProfileStore | None = None,
) -> APIRouter:
router = APIRouter()
@router.get("/api/v1/observatory/ai-module-catalog")
def catalog() -> dict[str, object]:
return {
**compositions.registry.catalog(),
"authority": {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
},
}
@router.get("/api/v1/observatory/lab-view-profiles/{result_id}")
def get_lab_view_profile(result_id: str) -> dict[str, object]:
if view_profiles is None:
raise HTTPException(503, "Профили отображения LAB недоступны.")
try:
profile = view_profiles.get(result_id)
except LabViewProfileError as exc:
raise HTTPException(422, "Некорректный профиль отображения LAB.") from exc
if profile is None:
raise HTTPException(404, "Профиль отображения LAB ещё не сохранён.")
return profile.as_dict()
@router.put("/api/v1/observatory/lab-view-profiles/{result_id}")
def put_lab_view_profile(result_id: str, request: LabViewProfileRequest) -> dict[str, object]:
if view_profiles is None:
raise HTTPException(503, "Профили отображения LAB недоступны.")
if request.schema_version != LAB_VIEW_PROFILE_SCHEMA or request.result_id != result_id:
raise HTTPException(422, "Профиль отображения относится к другой LAB.")
try:
settings = request.scene_settings
profile = LabViewProfile(
result_id=result_id,
scene_settings=LabSceneProfile(**settings.model_dump()),
updated_at_utc=datetime.now(UTC)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z"),
)
return view_profiles.save(profile).as_dict()
except LabViewProfileError as exc:
raise HTTPException(422, "Некорректные настройки отображения LAB.") from exc
@router.post("/api/v1/observatory/ai-compositions")
def save(request: AICompositionRequest) -> dict[str, object]:
if request.schema_version != COMPOSITION_SCHEMA:
raise HTTPException(
status_code=422,
detail="Версия конфигурации AI-слоя не поддерживается.",
)
try:
source = store.get_session(request.source_session_id).summary
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail="Запись Обсерватории не найдена.") from exc
if source.lab is not None:
raise HTTPException(
status_code=409,
detail="AI-слой настраивается для исходной записи.",
)
selection = {"schema_version": request.schema_version, "selections": request.selections}
try:
composition, created = compositions.save(selection)
except CompositionError as exc:
raise HTTPException(status_code=409, detail=_composition_error_detail(exc)) from exc
selected_modules = tuple(
node.module.module_id
for node in composition.nodes
if node.module.group != "preparation"
)
selected = set(selected_modules)
for previous in (
()
if composition_runs is None
else composition_runs.list(source_session_id=source.session_id)
):
if previous.composition_sha256 != composition.sha256:
continue
try:
previous_jobs = (
tuple(queue.get(job_id) for job_id in previous.job_ids) if queue else ()
)
except (ObservatoryRecordedQueueError, ValueError):
previous_jobs = ()
if previous_jobs and all(
job.state != "failed" and job.publication_state != "failed" for job in previous_jobs
):
raise HTTPException(
status_code=409,
detail=(
"Эта конфигурация уже рассчитана или поставлена в очередь. "
"Выберите другую конфигурацию."
),
)
setup_ids: list[str] = []
for module_id in ("ddrnet", "eomt", "tgs"):
if module_id in selected:
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS[module_id])
if "object-distance" in selected:
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS["object-distance"])
elif "rf-detr" in selected:
setup_ids.append(_EXECUTABLE_SINGLE_MODULE_SETUPS["rf-detr"])
jobs = []
reason = "Для этой композиции ещё не установлен исполняемый пакет Worker 006."
if setup_ids and definitions is not None and binding is not None:
try:
checked = []
existing_by_setup = {}
for setup_id in setup_ids:
definition = definitions.resolve_setup(setup_id)
existing = None
if queue is not None:
candidates = queue.list_jobs(
source_session_id=source.session_id,
setup_id=setup_id,
definition_sha256=definition.definition_sha256,
limit=20,
)
existing = next(
(job for job in candidates if job.state != "failed"),
None,
)
if existing is not None:
existing_by_setup[setup_id] = existing
continue
try:
check = binding.check(
source_session_id=source.session_id,
setup_id=setup_id,
definition_sha256=definition.definition_sha256,
)
except PortableSourceNotPreparedError:
check = binding.prepare_check(
source_session_id=source.session_id,
setup_id=setup_id,
definition_sha256=definition.definition_sha256,
)
checked.append((setup_id, definition, check))
for setup_id, definition, check in checked:
key = hashlib.sha256(
f"{request.idempotency_key}\0{setup_id}".encode()
).hexdigest()
job, _created = binding.submit(
source_session_id=source.session_id,
setup_id=setup_id,
definition_sha256=definition.definition_sha256,
expected_check_sha256=check.check_sha256,
idempotency_key=f"ai-layer:{key}",
)
existing_by_setup[setup_id] = job
jobs = [existing_by_setup[setup_id] for setup_id in setup_ids]
if composition_runs is None and not checked:
raise ObservatoryRecordedQueueDuplicateError("existing-composition")
reason = (
"Недостающие модули поставлены в очередь Worker 006; "
"готовые результаты использованы повторно."
if len(checked) < len(setup_ids)
else "Композиция поставлена в очередь Worker 006."
)
except ObservatoryRecordedQueueDuplicateError as exc:
raise HTTPException(
status_code=409,
detail=(
"Эта конфигурация уже рассчитана или поставлена в очередь. "
"Выберите другую конфигурацию."
),
) from exc
except (PortableQueueBindingError, ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(
status_code=409,
detail="Композицию не удалось поставить в очередь Worker 006.",
) from exc
if not jobs and definitions is not None and binding is not None:
raise HTTPException(
status_code=409,
detail="Для этой конфигурации пока нет исполняемых модулей Worker 006.",
)
run_projection: dict[str, object] | None = None
try:
if composition_runs is None or ontology is None or not jobs:
raise StopIteration
run = composition_runs.save(
source_session_id=source.session_id,
composition=composition,
setup_ids=tuple(setup_ids),
job_ids=tuple(job.job_id for job in jobs),
idempotency_key=request.idempotency_key,
created_at_utc=datetime.now(UTC)
.isoformat(timespec="milliseconds")
.replace("+00:00", "Z"),
)
presentation = ontology.project_composition(composition)
run_projection = {**run.as_dict(), "presentation": presentation}
except StopIteration:
pass
except (CompositionRunError, ObservatoryOntologyError) as exc:
raise HTTPException(
status_code=409,
detail="Связь композиции с результатами не удалось сохранить.",
) from exc
return {
"schema_version": (
"missioncore.observatory-ai-composition-receipt/v3"
if run_projection is not None
else "missioncore.observatory-ai-composition-receipt/v2"
),
"source_session_id": source.session_id,
"composition": composition.as_dict(),
"composition_sha256": composition.sha256,
"created": created,
**({"run": run_projection} if run_projection is not None else {}),
"dispatch": {
"ready": len(jobs) == len(setup_ids) and bool(jobs),
"reason": reason,
"setup_ids": setup_ids,
"jobs": [job.as_dict() for job in jobs],
},
}
def project_run(run: CompositionRun) -> dict[str, object]:
if ontology is None:
raise CompositionRunError("composition ontology is unavailable")
try:
exact = composition_runs.get(run.run_id)
jobs = [queue.get(job_id) for job_id in exact.job_ids] if queue else []
except (CompositionRunError, ObservatoryRecordedQueueError, ValueError):
raise
# The immutable composition document is the authority for projection;
# reconstruct the selected module projection from the run's sealed IDs.
presentation = ontology.project_module_ids(exact.module_ids)
published = bool(jobs) and all(
job.state == "succeeded" and job.publication_state == "published" and job.result_id
for job in jobs
)
failed = any(job.state == "failed" or job.publication_state == "failed" for job in jobs)
return {
**exact.as_dict(),
"state": "ready" if published else "failed" if failed else "running",
"configuration_label": (
f"{store.get_session(exact.source_session_id).summary.display_name} · "
f"{presentation['configuration_label']}"
),
"display_name": composition_runs.display_name(exact.run_id),
"presentation": presentation,
"jobs": [job.as_dict() for job in jobs],
"result_ids": [job.result_id for job in jobs if job.result_id is not None],
}
@router.get("/api/v1/observatory/ai-composition-runs")
def composition_run_list(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
) -> dict[str, object]:
try:
items = (
[]
if composition_runs is None
else [
project_run(run)
for run in composition_runs.list(
source_session_id=source_session_id,
include_hidden=False,
)
]
)
except (
CompositionRunError,
ObservatoryRecordedQueueError,
ObservatoryOntologyError,
ValueError,
) as exc:
raise HTTPException(503, "Композиции AI-слоя недоступны.") from exc
return {
"schema_version": "missioncore.observatory-ai-composition-run-list/v1",
"items": items,
}
@router.patch("/api/v1/observatory/ai-composition-runs/{run_id}")
def rename_composition_run_projection(
run_id: str,
request: AICompositionRunRenameRequest,
) -> dict[str, object]:
if request.schema_version != "missioncore.observatory-ai-composition-run-rename/v1":
raise HTTPException(422, "Версия переименования результата не поддерживается.")
if composition_runs is None:
raise HTTPException(503, "Композиции AI-слоя недоступны.")
try:
display_name = composition_runs.rename_projection(run_id, request.display_name)
except CompositionRunError as exc:
raise HTTPException(404, "Результат AI inference не найден.") from exc
return {
"schema_version": "missioncore.observatory-ai-composition-run-projection/v1",
"run_id": run_id,
"display_name": display_name,
}
@router.delete(
"/api/v1/observatory/ai-composition-runs/{run_id}",
status_code=204,
)
def delete_composition_run_projection(run_id: str) -> Response:
if composition_runs is None:
raise HTTPException(503, "Композиции AI-слоя недоступны.")
try:
composition_runs.delete_projection(run_id)
except CompositionRunError as exc:
raise HTTPException(404, "Результат AI inference не найден.") from exc
return Response(status_code=204)
@router.get("/api/v1/observatory/ai-runs")
def runs(
source_session_id: str = Query(
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$",
),
) -> dict[str, object]:
if queue is None:
raise HTTPException(status_code=503, detail="Очередь AI-слоёв недоступна.")
try:
jobs = [
job
for setup_id in _EXECUTABLE_SINGLE_MODULE_SETUPS.values()
for job in queue.list_jobs(
source_session_id=source_session_id,
setup_id=setup_id,
limit=20,
)
]
except (ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(status_code=503, detail="Очередь AI-слоёв недоступна.") from exc
jobs.sort(key=lambda job: job.created_at_utc, reverse=True)
return {
"schema_version": "missioncore.observatory-recorded-job-list/v1",
"items": [job.as_dict() for job in jobs[:20]],
"authority": {
"commands_enabled": False,
"actuation_allowed": False,
"navigation_or_safety_accepted": False,
"production_accepted": False,
},
}
return router
+101 -2
View File
@@ -9,15 +9,33 @@ from fastapi import APIRouter, HTTPException, Path, Query, Response
from fastapi.responses import FileResponse
from k1link.laboratory.canonical_rerun_overlay import CanonicalLabOverlayError
from k1link.observatory.composition_runs import CompositionRunError, CompositionRunStore
from k1link.observatory.portable_replay import PortableReplayService
from k1link.observatory.portable_result_view import PortableResultViewError
from k1link.observatory.recorded_jobs import (
ObservatoryRecordedJobQueue,
ObservatoryRecordedQueueError,
)
ResultId = Annotated[str, Path(pattern=r"^m49-tgs-portable-review-[a-f0-9]{64}$")]
ResultId = Annotated[
str,
Path(
pattern=(
r"^(?:m49-tgs-portable-review|lab-v1-eomt-ddrnet|"
r"ai-layer-(?:ddrnet|eomt|rf-detr|object-distance))-[a-f0-9]{64}$"
)
),
]
BaseSha = Annotated[str, Path(pattern=r"^[a-f0-9]{64}$")]
_LOG = logging.getLogger(__name__)
def build_portable_replay_router(service: PortableReplayService) -> APIRouter:
def build_portable_replay_router(
service: PortableReplayService,
*,
composition_runs: CompositionRunStore | None = None,
queue: ObservatoryRecordedJobQueue | None = None,
) -> APIRouter:
router = APIRouter(tags=["observatory"])
path = "/api/v1/observatory/portable-results/{result_id}/replays/{base_sha}/recording.rrd"
@@ -72,4 +90,85 @@ def build_portable_replay_router(service: PortableReplayService) -> APIRouter:
},
)
composition_path = (
"/api/v1/observatory/ai-composition-runs/{run_id}/replays/{base_sha}/recording.rrd"
)
def composition_members(run_id: str) -> tuple[str, ...]:
if composition_runs is None or queue is None:
raise HTTPException(503, "Составной просмотр AI-слоя недоступен.")
try:
run = composition_runs.get(run_id)
jobs = tuple(queue.get(job_id) for job_id in run.job_ids)
except (CompositionRunError, ObservatoryRecordedQueueError, ValueError) as exc:
raise HTTPException(409, "Составной запуск AI-слоя недоступен.") from exc
if not jobs or any(
job.state != "succeeded"
or job.publication_state != "published"
or job.result_id is None
for job in jobs
):
raise HTTPException(409, "Расчёт всех модулей этой конфигурации ещё не завершён.")
return tuple(job.result_id for job in jobs if job.result_id is not None)
@router.head(composition_path)
def prepare_composition(
run_id: Annotated[str, Path(pattern=r"^ai-composition-[a-f0-9]{64}$")],
base_sha: BaseSha,
) -> Response:
try:
artifact = service.prepare_composition(run_id, composition_members(run_id), base_sha)
except (
ValueError,
OSError,
KeyError,
TypeError,
CanonicalLabOverlayError,
PortableResultViewError,
) as exc:
_LOG.exception("Composition replay packaging rejected")
raise HTTPException(
409, "Составной результат не удалось подготовить к просмотру."
) from exc
return Response(
media_type="application/vnd.rerun.rrd",
headers={
"Content-Length": str(artifact.byte_length),
"ETag": f'"{artifact.sha256}"',
"X-Rerun-Format": "RRF2",
"Cache-Control": "private, no-store",
},
)
@router.get(composition_path)
def read_composition(
run_id: Annotated[str, Path(pattern=r"^ai-composition-[a-f0-9]{64}$")],
base_sha: BaseSha,
generation: Annotated[str, Query(pattern=r"^[a-f0-9]{64}$")],
) -> FileResponse:
try:
artifact = service.cached_composition(run_id, composition_members(run_id), base_sha)
except (
ValueError,
OSError,
KeyError,
TypeError,
PortableResultViewError,
) as exc:
raise HTTPException(409, "Кэш составного результата не прошёл проверку.") from exc
if artifact is None:
raise HTTPException(409, "Составной просмотр ещё не подготовлен.")
if artifact.sha256 != generation:
raise HTTPException(412, "Версия составного просмотра изменилась.")
return FileResponse(
artifact.path,
media_type="application/vnd.rerun.rrd",
headers={
"ETag": f'"{artifact.sha256}"',
"X-Rerun-Format": "RRF2",
"Cache-Control": "private, max-age=31536000, immutable",
"X-Content-Type-Options": "nosniff",
},
)
return router
+117 -13
View File
@@ -49,7 +49,10 @@ from k1link.viewer.recorded import (
from k1link.viewer.recorded import (
RecordedBlueprintError,
recorded_blueprint_rrd,
recorded_blueprint_sessions,
)
from k1link.viewer.recorded_blueprint_lifecycle import BlueprintSessionReleased
from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit
from k1link.viewer.rerun_bridge import RerunSceneSettings
DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
@@ -102,7 +105,7 @@ class ReplayRequest(StrictApiModel):
loop: bool = False
class RecordedBlueprintRequest(StrictApiModel):
class RecordedBlueprintIdentity(StrictApiModel):
application_id: Literal["nodedc_mission_core_recorded"]
recording_id: str = Field(
min_length=1,
@@ -114,25 +117,58 @@ class RecordedBlueprintRequest(StrictApiModel):
max_length=32,
pattern=r"^[a-f0-9]{32}$",
)
accumulation_seconds: float = Field(strict=True, ge=0.0, le=3600.0)
class RecordedBlueprintLifecycleRequest(RecordedBlueprintIdentity):
action: Literal["renew", "release"]
EyeCoordinate = Annotated[float, Field(strict=True, allow_inf_nan=False)]
EyeVector = tuple[EyeCoordinate, EyeCoordinate, EyeCoordinate]
class RecordedBlueprintRequest(RecordedBlueprintIdentity):
accumulation_seconds: float = Field(strict=True, ge=0.0, allow_inf_nan=False)
show_points: StrictBool
show_trajectory: StrictBool
show_grid: StrictBool
point_size: float = Field(default=2.5, strict=True, ge=0.1, le=32.0)
point_size: float = Field(default=2.5, strict=True, ge=0.1, allow_inf_nan=False)
color_mode: Literal["intensity", "height", "distance", "rgb", "class"] = "intensity"
palette: Literal["turbo", "viridis", "plasma", "grayscale", "custom"] = "turbo"
custom_color: str = Field(default="#f7f8f4", pattern=r"^#[0-9A-Fa-f]{6}$")
active_view: Literal["spatial", "perception", "perception3d", "metrics"] = "spatial"
view_reset_generation: Literal[0, 1] = 0
unified_perception: StrictBool = False
unified_camera_share: float = Field(strict=True, ge=0.1, le=0.9, default=0.46)
semantic_layer: Literal["city", "vegetation"] | None = None
plan_view: StrictBool = False
show_detections_2d: StrictBool = False
show_camera_image: StrictBool = True
show_segmentation: StrictBool = False
show_cuboids_3d: StrictBool = False
show_costmap: StrictBool = False
reactivate_updates: StrictBool = False
follow_trajectory: StrictBool = False
eye_position: EyeVector | None = None
eye_look_target: EyeVector | None = None
eye_up: EyeVector | None = None
current_time_ns: int | None = Field(default=None, strict=True, ge=0, le=MAX_SAFE_INTEGER)
@model_validator(mode="after")
def validate_eye_vectors(self) -> RecordedBlueprintRequest:
vectors = (self.eye_position, self.eye_look_target, self.eye_up)
if any(vector is None for vector in vectors):
if not all(vector is None for vector in vectors):
raise ValueError("all eye vectors must be supplied together")
return self
assert self.eye_position is not None
assert self.eye_look_target is not None
assert self.eye_up is not None
if self.eye_position == self.eye_look_target:
raise ValueError("eye position and look target must differ")
if sum(value * value for value in self.eye_up) <= 1.0e-12:
raise ValueError("eye up vector must be non-zero")
return self
class RecordedPerceptionRequest(StrictApiModel):
@@ -365,6 +401,7 @@ def build_session_router(
cursor: str | None = Query(default=None, max_length=128),
scope: Literal["all", "source", "laboratory"] = "all",
lab_contract: Literal["v1", "v2", "v3"] = "v1",
pagination: Literal["cursor-v1"] | None = None,
) -> dict[str, Any]:
try:
_refresh_catalog(catalog_refresher)
@@ -375,6 +412,14 @@ def build_session_router(
include_capability_projections=lab_contract in ("v2", "v3"),
)
return {
**(
{
"schema_version": "missioncore.observation-session-page/v1",
"next_cursor": page.next_cursor,
}
if pagination == "cursor-v1"
else {}
),
"items": [
{
"id": item.session_id,
@@ -391,9 +436,7 @@ def build_session_router(
else item.capture_attestation.as_dict()
),
**(
{
"lab": lab_catalog_document(item, lab_contract)
}
{"lab": lab_catalog_document(item, lab_contract)}
if item.lab is not None
else {}
),
@@ -411,7 +454,7 @@ def build_session_router(
}
for item in page.items
if item.started_at_utc is not None
]
],
}
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@@ -870,9 +913,7 @@ def build_session_router(
**response_kwargs,
)
@router.get(
"/api/v1/observation-sessions/{session_id}/canonical-lab/spatial-frame"
)
@router.get("/api/v1/observation-sessions/{session_id}/canonical-lab/spatial-frame")
async def get_observation_session_canonical_lab_spatial_frame(
session_id: str,
generation: Annotated[str, Query(min_length=64, max_length=64)],
@@ -937,20 +978,37 @@ def build_session_router(
headers={
"Cache-Control": "private, max-age=31536000, immutable",
"ETag": (
f'"{generation}:{CANONICAL_LAB_SPATIAL_PROFILE}:'
f'{payload["source_time_ns"]}"'
f'"{generation}:{CANONICAL_LAB_SPATIAL_PROFILE}:{payload["source_time_ns"]}"'
),
"X-Content-Type-Options": "nosniff",
},
)
@router.post("/api/v1/observation-sessions/{session_id}/blueprint-lifecycle")
async def update_recorded_blueprint_lifecycle(
session_id: str,
request: RecordedBlueprintLifecycleRequest,
) -> Response:
# Releasing an ephemeral owner must still work after source removal.
# The random viewport identity authorizes only its own memory resource;
# this route never materializes or deletes recordings/artifacts.
if not SAFE_SOURCE_ID.fullmatch(session_id):
raise HTTPException(status_code=422, detail="Некорректный идентификатор сессии.")
key = (request.application_id, request.recording_id, request.blueprint_session_id)
if request.action == "release":
await run_in_threadpool(recorded_blueprint_sessions.release, key)
else:
await run_in_threadpool(recorded_blueprint_sessions.renew, key)
return Response(status_code=204, headers={"Cache-Control": "no-store"})
@router.post("/api/v1/observation-sessions/{session_id}/blueprint.rrd")
async def get_observation_session_blueprint(
session_id: str,
request: RecordedBlueprintRequest,
) -> Response:
camera_max_orbital_radius: float | None = None
try:
await run_in_threadpool(
command = await run_in_threadpool(
_prepare_replay,
store,
catalog_refresher,
@@ -977,19 +1035,55 @@ def build_session_router(
active_view=request.active_view,
view_reset_generation=request.view_reset_generation,
unified_perception=request.unified_perception,
unified_camera_share=request.unified_camera_share,
semantic_layer=request.semantic_layer,
plan_view=request.plan_view,
show_detections_2d=request.show_detections_2d,
show_camera_image=request.show_camera_image,
show_segmentation=request.show_segmentation,
show_cuboids_3d=request.show_cuboids_3d,
show_costmap=request.show_costmap,
reactivate_updates=request.reactivate_updates,
follow_trajectory=request.follow_trajectory,
eye_position=request.eye_position,
eye_look_target=request.eye_look_target,
eye_up=request.eye_up,
)
if request.current_time_ns is not None:
camera_recording = None
if recording_preparation_manager is not None:
snapshot = recording_preparation_manager.status(session_id)
if (
snapshot is not None
and snapshot.state == "ready"
and snapshot.recording is not None
):
camera_recording = snapshot.recording
# A composition replay consumes the immutable base launch but
# does not GET its recording. Its short launch reservation can
# therefore expire while the combined RRD remains open. Restore
# the already-published base descriptor with bounded stat checks
# so later layer toggles still receive the native zoom limit.
if camera_recording is None and recording_materializer is not None:
camera_recording = await run_in_threadpool(
recording_materializer.restore_published,
command,
)
if camera_recording is not None:
camera_max_orbital_radius = await run_in_threadpool(
recorded_orbital_radius_limit,
camera_recording.path,
current_time_ns=request.current_time_ns,
accumulation_seconds=request.accumulation_seconds,
show_points=request.show_points,
show_trajectory=request.show_trajectory,
)
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except (SessionNotReplayableError, SessionIntegrityError) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
except BlueprintSessionReleased as exc:
raise HTTPException(status_code=410, detail="Сессия визуализатора закрыта.") from exc
except RecordedBlueprintError as exc:
raise HTTPException(
status_code=500,
@@ -1007,6 +1101,16 @@ def build_session_router(
"Cache-Control": "no-store",
"X-Content-Type-Options": "nosniff",
"Content-Disposition": 'inline; filename="blueprint.rrd"',
**(
{
"X-MissionCore-Camera-Max-Orbital-Radius": format(
camera_max_orbital_radius,
".9g",
)
}
if camera_max_orbital_radius is not None
else {}
),
},
)