feat(observatory): admit canonical recorded replay
This commit is contained in:
@@ -0,0 +1,670 @@
|
||||
"""Publish an immutable canonical recorded LAB result into the Session catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.laboratory.evidence_registry import LaboratoryEvidenceDefinition
|
||||
from k1link.laboratory.evidence_report import (
|
||||
LaboratoryEvidenceReportError,
|
||||
verify_laboratory_evidence_result,
|
||||
)
|
||||
from k1link.laboratory.vegetation_shadow_lab import LAB_SCHEMA
|
||||
from k1link.sessions import LabReplayCapability, LabSessionBinding, SessionStore
|
||||
from k1link.sessions.models import SessionDetail, SessionStoreError
|
||||
|
||||
CANONICAL_RECORDED_PROJECTION_SCHEMA: Final = (
|
||||
"missioncore.canonical-recorded-lab-projection/v1"
|
||||
)
|
||||
CANONICAL_REPLAY_CAPABILITY_SCHEMA: Final = (
|
||||
"missioncore.observation-lab-replay-capability/v1"
|
||||
)
|
||||
CANONICAL_SOURCE_SESSION_ID: Final = "20260828T130511Z_viewer_live"
|
||||
CANONICAL_SOURCE_LABEL: Final = "RAVNOVES004TREE"
|
||||
CANONICAL_WORK_ID: Final = "lab-v1-vegetation-shadow"
|
||||
CANONICAL_RESULT_PREFIX: Final = "lab-v1-vegetation-shadow"
|
||||
_MAX_DOCUMENT_BYTES: Final = 1024 * 1024
|
||||
_RESULT_ID = re.compile(r"^lab-v1-vegetation-shadow-[a-f0-9]{64}$")
|
||||
_DEFINITION: Final = LaboratoryEvidenceDefinition(
|
||||
work_id=CANONICAL_WORK_ID,
|
||||
runtime_relative_root=PurePosixPath("lab-v1-vegetation/results"),
|
||||
result_id_prefix=CANONICAL_RESULT_PREFIX,
|
||||
document_name="result.json",
|
||||
result_schema_version=LAB_SCHEMA,
|
||||
)
|
||||
|
||||
|
||||
class CanonicalRecordedCatalogError(ValueError):
|
||||
"""The sealed result cannot be admitted to the recorded Session catalog."""
|
||||
|
||||
|
||||
def publish_canonical_recorded_vegetation_result(
|
||||
*,
|
||||
store: SessionStore,
|
||||
runtime_root: Path,
|
||||
result_root: Path,
|
||||
) -> LabSessionBinding:
|
||||
"""Project one exact, verified full-route RAV004 result without computing replay data."""
|
||||
|
||||
runtime = _real_directory(runtime_root, "LAB runtime root")
|
||||
candidate = _real_directory(result_root, "canonical LAB result")
|
||||
if _RESULT_ID.fullmatch(candidate.name) is None:
|
||||
raise CanonicalRecordedCatalogError("canonical LAB result identity is invalid")
|
||||
expected_parent = _real_directory(
|
||||
_DEFINITION.result_root(runtime),
|
||||
"canonical LAB result collection",
|
||||
)
|
||||
if not expected_parent.is_relative_to(runtime):
|
||||
raise CanonicalRecordedCatalogError(
|
||||
"canonical LAB result collection escaped its runtime root"
|
||||
)
|
||||
if candidate.parent != expected_parent:
|
||||
raise CanonicalRecordedCatalogError("canonical LAB result escaped its registered root")
|
||||
|
||||
try:
|
||||
proof = verify_laboratory_evidence_result(_DEFINITION, candidate)
|
||||
except LaboratoryEvidenceReportError as exc:
|
||||
raise CanonicalRecordedCatalogError(str(exc)) from exc
|
||||
document = _read_document(
|
||||
candidate / _DEFINITION.document_name,
|
||||
expected_sha256=str(proof["document_sha256"]),
|
||||
)
|
||||
result_id = _exact_text(document.get("result_id"), candidate.name, "result id")
|
||||
_exact_text(document.get("schema_version"), LAB_SCHEMA, "result schema")
|
||||
_exact_text(
|
||||
document.get("status"),
|
||||
"visual-shadow-ready-policy-not-authorized",
|
||||
"result status",
|
||||
)
|
||||
identity_sha256 = _exact_text(
|
||||
document.get("identity_sha256"),
|
||||
str(proof["identity_sha256"]),
|
||||
"identity digest",
|
||||
)
|
||||
authority = _object(document.get("authority"), "result authority")
|
||||
_require_observation_only(authority, "result authority")
|
||||
identity = _object(document.get("identity"), "result identity")
|
||||
identity_authority = _object(identity.get("authority"), "identity authority")
|
||||
_require_observation_only(identity_authority, "identity authority")
|
||||
if identity_authority != authority:
|
||||
raise CanonicalRecordedCatalogError("top-level authority is not identity-bound")
|
||||
|
||||
if document.get("route_video") is not None or document.get("route_review") is not None:
|
||||
raise CanonicalRecordedCatalogError("canonical full-route result shape is invalid")
|
||||
review = _object(document.get("route_full_review"), "full-route review")
|
||||
identity_review = _object(
|
||||
identity.get("route_full_review"),
|
||||
"identity full-route review",
|
||||
)
|
||||
if identity_review != review:
|
||||
raise CanonicalRecordedCatalogError("full-route review is not identity-bound")
|
||||
_exact_text(review.get("source_id"), CANONICAL_SOURCE_LABEL, "source label")
|
||||
source_session_id = _exact_text(
|
||||
review.get("session_id"),
|
||||
CANONICAL_SOURCE_SESSION_ID,
|
||||
"source session id",
|
||||
)
|
||||
if review.get("frame_count") != 6830:
|
||||
raise CanonicalRecordedCatalogError("canonical frame count is invalid")
|
||||
source_result_id = _text(
|
||||
review.get("linked_route_review_result_id"),
|
||||
"linked route review result id",
|
||||
)
|
||||
if _RESULT_ID.fullmatch(source_result_id) is None:
|
||||
raise CanonicalRecordedCatalogError("linked route review identity is invalid")
|
||||
if identity.get("base_result_id") != source_result_id:
|
||||
raise CanonicalRecordedCatalogError("base result is not identity-bound")
|
||||
result_source = _object(document.get("source"), "result source")
|
||||
if result_source != _object(
|
||||
identity.get("source"),
|
||||
"identity source",
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("result source is not identity-bound")
|
||||
if result_source != {
|
||||
"shadow_session": CANONICAL_SOURCE_LABEL,
|
||||
"shadow_camera": "sensor.camera.right",
|
||||
"shadow_frame_count": 6830,
|
||||
"video_shadow_frame_count": 6830,
|
||||
}:
|
||||
raise CanonicalRecordedCatalogError("canonical source identity changed")
|
||||
timeline_start = _finite_number(
|
||||
review.get("timeline_start_seconds"),
|
||||
"timeline start",
|
||||
)
|
||||
timeline_end = _finite_number(
|
||||
review.get("timeline_end_seconds"),
|
||||
"timeline end",
|
||||
)
|
||||
if timeline_end <= timeline_start:
|
||||
raise CanonicalRecordedCatalogError("canonical timeline range is invalid")
|
||||
_validate_viewer_contract(candidate, document, review)
|
||||
try:
|
||||
source_detail, source_catalog_sha256 = (
|
||||
store.get_session_with_catalog_snapshot(source_session_id)
|
||||
)
|
||||
except SessionStoreError as exc:
|
||||
raise CanonicalRecordedCatalogError(
|
||||
"canonical source session is unavailable"
|
||||
) from exc
|
||||
_validate_source_catalog_binding(source_detail)
|
||||
|
||||
sealed_method = _object(document.get("method"), "sealed method")
|
||||
pipeline_id = _exact_text(
|
||||
sealed_method.get("pipeline_id"),
|
||||
"ravnoves004tree-full-eomt-ddrnet-recorded-review/v1",
|
||||
"sealed pipeline id",
|
||||
)
|
||||
execution_class = _exact_text(
|
||||
sealed_method.get("execution_class"),
|
||||
"ai-inference",
|
||||
"sealed execution class",
|
||||
)
|
||||
capability = LabReplayCapability(
|
||||
schema_version=CANONICAL_REPLAY_CAPABILITY_SCHEMA,
|
||||
kind="canonical-recorded-rerun",
|
||||
viewer_profile="recorded-session",
|
||||
timeline="session_time",
|
||||
activation="explicit",
|
||||
commands_enabled=False,
|
||||
)
|
||||
provenance = {
|
||||
"schema_version": CANONICAL_RECORDED_PROJECTION_SCHEMA,
|
||||
"evidence_identity_sha256": identity_sha256,
|
||||
"result_document_sha256": proof["document_sha256"],
|
||||
"replay_capability": capability.as_dict(),
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
},
|
||||
"method": {
|
||||
"schema_version": "missioncore.laboratory-method/v1",
|
||||
"completeness": "legacy-partial",
|
||||
"execution_class": execution_class,
|
||||
"pipeline_id": pipeline_id,
|
||||
"components": [
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "sealed full-route LAB result",
|
||||
"version": LAB_SCHEMA,
|
||||
"role": "immutable Session catalog projection",
|
||||
"identity_sha256": identity_sha256,
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
return store.publish_lab_instance(
|
||||
session_id=result_id,
|
||||
source_session_id=source_session_id,
|
||||
display_name="RAVNOVES004TREE · полный маршрут восприятия",
|
||||
lab_id="LAB V1",
|
||||
result_kind="recorded-perception-qualification",
|
||||
result_id=result_id,
|
||||
source_result_id=source_result_id,
|
||||
config_sha256=None,
|
||||
run_created_at_utc=_text(document.get("created_at_utc"), "creation time"),
|
||||
replay_capability=capability,
|
||||
provenance=provenance,
|
||||
duration_seconds=timeline_end - timeline_start,
|
||||
include_recorded_media=False,
|
||||
expected_source_catalog_sha256=source_catalog_sha256,
|
||||
)
|
||||
|
||||
|
||||
def _real_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
raise CanonicalRecordedCatalogError(f"{label} must not be a symlink")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise CanonicalRecordedCatalogError(f"{label} is unavailable") from exc
|
||||
if not resolved.is_dir():
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be a directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _read_document(path: Path, *, expected_sha256: str) -> dict[str, Any]:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > _MAX_DOCUMENT_BYTES:
|
||||
raise CanonicalRecordedCatalogError("canonical LAB document is unavailable")
|
||||
try:
|
||||
payload = path.read_bytes()
|
||||
if hashlib.sha256(payload).hexdigest() != expected_sha256:
|
||||
raise CanonicalRecordedCatalogError(
|
||||
"canonical LAB document changed after verification"
|
||||
)
|
||||
value = json.loads(payload)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise CanonicalRecordedCatalogError("canonical LAB document is invalid") from exc
|
||||
return _object(value, "canonical LAB document")
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _text(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _exact_text(value: object, expected: str, label: str) -> str:
|
||||
text = _text(value, label)
|
||||
if text != expected:
|
||||
raise CanonicalRecordedCatalogError(f"{label} changed")
|
||||
return text
|
||||
|
||||
|
||||
def _finite_number(value: object, label: str) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be numeric")
|
||||
number = float(value)
|
||||
if not math.isfinite(number):
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be finite")
|
||||
return number
|
||||
|
||||
|
||||
def _require_observation_only(value: dict[str, Any], label: str) -> None:
|
||||
expected = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
}
|
||||
if set(value) != set(expected) or any(value[key] is not False for key in expected):
|
||||
raise CanonicalRecordedCatalogError(f"{label} permits control authority")
|
||||
|
||||
|
||||
def _validate_viewer_contract(
|
||||
result_root: Path,
|
||||
document: dict[str, Any],
|
||||
review: dict[str, Any],
|
||||
) -> None:
|
||||
"""Require the metadata and artifacts consumed before the shared viewer mounts."""
|
||||
|
||||
if document.get("ground_truth") is not False:
|
||||
raise CanonicalRecordedCatalogError("canonical result ground-truth marker changed")
|
||||
identity = _object(document.get("identity"), "result identity")
|
||||
selected_candidate = _text(
|
||||
identity.get("selected_candidate"),
|
||||
"selected candidate",
|
||||
)
|
||||
if selected_candidate not in {"ddrnet", "ppliteseg"}:
|
||||
raise CanonicalRecordedCatalogError("canonical selected candidate changed")
|
||||
|
||||
metrics = _object(document.get("metrics"), "result metrics")
|
||||
candidates = _object(metrics.get("candidates"), "candidate metrics")
|
||||
identity_candidates = _object(
|
||||
identity.get("candidate_metrics"),
|
||||
"identity candidate metrics",
|
||||
)
|
||||
if candidates != identity_candidates or set(candidates) != {"ddrnet", "ppliteseg"}:
|
||||
raise CanonicalRecordedCatalogError("candidate metrics are not identity-bound")
|
||||
for candidate in ("ddrnet", "ppliteseg"):
|
||||
_validate_candidate_metrics(
|
||||
_object(candidates.get(candidate), f"{candidate} metrics"),
|
||||
candidate,
|
||||
)
|
||||
|
||||
decision = _object(document.get("decision"), "result decision")
|
||||
if (
|
||||
decision.get("selected_candidate") != selected_candidate
|
||||
or decision.get("visual_shadow_ready") is not True
|
||||
or decision.get("mission_policy_ready_for_configuration") is not True
|
||||
or decision.get("navigation_accepted") is not False
|
||||
or decision.get("production_accepted") is not False
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical result decision changed")
|
||||
catalogs = _object(document.get("catalogs"), "result catalogs")
|
||||
if catalogs.get("goose") != [] or catalogs.get("ravnoves") != []:
|
||||
raise CanonicalRecordedCatalogError("canonical result catalogs changed")
|
||||
limitations = document.get("limitations")
|
||||
if not isinstance(limitations, list) or not limitations or not all(
|
||||
isinstance(item, str) and item.strip() for item in limitations
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical result limitations are invalid")
|
||||
|
||||
expected_route_values: dict[str, object] = {
|
||||
"source_job_id": "recorded-camera-eb2783c5480d56bda07c8af0",
|
||||
"source_job_input_sha256": (
|
||||
"eb2783c5480d56bda07c8af008dff5344d19dc550ef70fe2075d6f098f7cc715"
|
||||
),
|
||||
"source_stream_sha256": (
|
||||
"e5eb017e2cc0f546736eda5235ca157b501913093cb64af5e548e335417e1bac"
|
||||
),
|
||||
"recorded_media_source_id": "recorded.camera.6a3945242828a038",
|
||||
"recorded_media_generation_sha256": (
|
||||
"b073ea1e7babf1c77a664e1a5b95e3702d0e05b0e34c1e85a7c67a6f8b392ded"
|
||||
),
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
}
|
||||
if any(review.get(key) != expected for key, expected in expected_route_values.items()):
|
||||
raise CanonicalRecordedCatalogError("canonical full-route source binding changed")
|
||||
if review.get("ground_truth") is not False:
|
||||
raise CanonicalRecordedCatalogError("canonical full-route ground-truth marker changed")
|
||||
|
||||
artifacts = _artifact_catalog(document.get("artifacts"))
|
||||
timeline = _object(review.get("timeline"), "full-route timeline")
|
||||
if (
|
||||
timeline.get("path") != "video/frame-source-times-ns.bin"
|
||||
or timeline.get("encoding") != "uint64-le-nanoseconds"
|
||||
or timeline.get("frame_count") != 6830
|
||||
or timeline.get("byte_length") != 6830 * 8
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical full-route timeline changed")
|
||||
timeline_sha256 = _sha256_text(timeline.get("sha256"), "timeline digest")
|
||||
_require_artifact(
|
||||
artifacts,
|
||||
path="video/frame-source-times-ns.bin",
|
||||
sha256=timeline_sha256,
|
||||
byte_length=6830 * 8,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
_validate_timeline_payload(
|
||||
result_root / "video" / "frame-source-times-ns.bin",
|
||||
expected_sha256=timeline_sha256,
|
||||
start_seconds=_finite_number(review.get("timeline_start_seconds"), "timeline start"),
|
||||
)
|
||||
|
||||
decode_repair = _object(review.get("decode_repair"), "decode repair")
|
||||
if (
|
||||
decode_repair.get("repaired_frame_count") != 1
|
||||
or decode_repair.get("sequence") != 6092
|
||||
or decode_repair.get("method") != "duplicate-previous-decoded-frame"
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical decode repair changed")
|
||||
repair_proofs = _object(decode_repair.get("proofs"), "decode repair proofs")
|
||||
for key, path in {
|
||||
"eomt": "proofs/decode_repair.json",
|
||||
"ddrnet": "proofs/ddrnet_decode_repair.json",
|
||||
}.items():
|
||||
proof = _object(repair_proofs.get(key), f"{key} decode proof")
|
||||
digest = _sha256_text(proof.get("sha256"), f"{key} decode proof digest")
|
||||
if proof.get("path") != path:
|
||||
raise CanonicalRecordedCatalogError("canonical decode proof changed")
|
||||
_require_artifact(artifacts, path=path, sha256=digest)
|
||||
|
||||
route_proofs = _object(review.get("proofs"), "full-route proofs")
|
||||
job_proof = _object(route_proofs.get("job"), "full-route job proof")
|
||||
job_digest = _sha256_text(job_proof.get("sha256"), "full-route job digest")
|
||||
if job_proof.get("path") != "proofs/job.json":
|
||||
raise CanonicalRecordedCatalogError("canonical job proof changed")
|
||||
_require_artifact(artifacts, path="proofs/job.json", sha256=job_digest)
|
||||
|
||||
layers = _object(review.get("layers"), "full-route layers")
|
||||
if set(layers) != {"city", "vegetation"}:
|
||||
raise CanonicalRecordedCatalogError("canonical full-route layers changed")
|
||||
_validate_full_route_layer(
|
||||
_object(layers.get("city"), "city layer"),
|
||||
layer="city",
|
||||
artifacts=artifacts,
|
||||
)
|
||||
_validate_full_route_layer(
|
||||
_object(layers.get("vegetation"), "vegetation layer"),
|
||||
layer="vegetation",
|
||||
artifacts=artifacts,
|
||||
)
|
||||
|
||||
|
||||
def _validate_source_catalog_binding(detail: SessionDetail) -> None:
|
||||
summary = detail.summary
|
||||
if (
|
||||
detail.plugin_id != "nodedc.device.xgrids-lixelkity-k1"
|
||||
or detail.archive_id != "xgrids-k1.viewer-live.evidence"
|
||||
or summary.session_id != CANONICAL_SOURCE_SESSION_ID
|
||||
or summary.display_name != CANONICAL_SOURCE_LABEL
|
||||
or summary.status != "ready"
|
||||
or summary.started_at_utc != "2026-08-28T13:05:16.249Z"
|
||||
or summary.completed_at_utc != "2026-08-28T13:18:45.030Z"
|
||||
or summary.duration_seconds is None
|
||||
or not math.isclose(summary.duration_seconds, 808.779495667, abs_tol=1e-9)
|
||||
or summary.modalities != ("point-cloud", "trajectory", "video")
|
||||
or summary.source_count != 3
|
||||
or summary.total_bytes != 799_020_963
|
||||
or summary.replayable is not True
|
||||
or summary.lab is not None
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical source catalog identity changed")
|
||||
sources = {
|
||||
source.source_id: (
|
||||
source.semantic_channel_id,
|
||||
source.modality,
|
||||
source.status,
|
||||
source.seekable,
|
||||
source.artifact_id,
|
||||
)
|
||||
for source in detail.sources
|
||||
}
|
||||
if sources != {
|
||||
"sensor.camera.right": (
|
||||
"camera.video.recorded",
|
||||
"video",
|
||||
"recorded",
|
||||
True,
|
||||
"recorded-video-6a3945242828a038",
|
||||
),
|
||||
"sensor.lidar.primary": (
|
||||
"spatial.point-cloud.recorded",
|
||||
"point-cloud",
|
||||
"recorded",
|
||||
True,
|
||||
"raw-transport-primary",
|
||||
),
|
||||
"spatial.trajectory": (
|
||||
"spatial.pose.recorded",
|
||||
"trajectory",
|
||||
"recorded",
|
||||
True,
|
||||
"raw-transport-primary",
|
||||
),
|
||||
}:
|
||||
raise CanonicalRecordedCatalogError("canonical source channels changed")
|
||||
artifacts = {artifact.artifact_id: artifact for artifact in detail.artifacts}
|
||||
if set(artifacts) != {
|
||||
"raw-transport-clock",
|
||||
"raw-transport-clock-origin",
|
||||
"raw-transport-index",
|
||||
"raw-transport-primary",
|
||||
"recorded-video-6a3945242828a038",
|
||||
}:
|
||||
raise CanonicalRecordedCatalogError("canonical source artifacts changed")
|
||||
raw = artifacts["raw-transport-primary"]
|
||||
video = artifacts["recorded-video-6a3945242828a038"]
|
||||
if (
|
||||
raw.kind != "raw-transport"
|
||||
or raw.media_type != "application/x-nodedc-k1mqtt"
|
||||
or raw.byte_length != 245_183_013
|
||||
or raw.sha256 != "20c789eff922a6bbb53592f86614abc0729a30544df29e740e7a378d12af85c2"
|
||||
or raw.integrity_status != "verified"
|
||||
or video.kind != "recorded-video"
|
||||
or video.media_type != "video/mp4"
|
||||
or video.byte_length != 553_837_950
|
||||
or video.sha256 is not None
|
||||
or video.integrity_status != "validated-structure"
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical source artifact proof changed")
|
||||
def _validate_candidate_metrics(value: dict[str, Any], candidate: str) -> None:
|
||||
_text(value.get("loaded_model_name"), f"{candidate} model name")
|
||||
_sha256_text(value.get("checkpoint_sha256"), f"{candidate} checkpoint")
|
||||
validation = _object(value.get("validation_metrics"), f"{candidate} validation")
|
||||
validation_timing = _object(
|
||||
value.get("validation_timing"),
|
||||
f"{candidate} validation timing",
|
||||
)
|
||||
shadow_timing = _object(value.get("shadow_timing"), f"{candidate} shadow timing")
|
||||
resource = _object(value.get("resource"), f"{candidate} resource")
|
||||
for key in ("mean_iou_percent", "published_mean_iou_percent", "vegetation_mean_iou"):
|
||||
_finite_number(validation.get(key), f"{candidate} {key}")
|
||||
for key in ("latency_ms_p95", "throughput_fps_from_mean_inference"):
|
||||
_finite_number(validation_timing.get(key), f"{candidate} validation {key}")
|
||||
_finite_number(shadow_timing.get(key), f"{candidate} shadow {key}")
|
||||
_finite_number(shadow_timing.get("prewarm_latency_ms"), f"{candidate} prewarm")
|
||||
_nonnegative_integer(resource.get("peak_reserved_vram_bytes"), f"{candidate} VRAM")
|
||||
_text(resource.get("gpu_name"), f"{candidate} GPU")
|
||||
|
||||
|
||||
def _validate_full_route_layer(
|
||||
value: dict[str, Any],
|
||||
*,
|
||||
layer: str,
|
||||
artifacts: dict[str, dict[str, Any]],
|
||||
) -> None:
|
||||
expected = {
|
||||
"city": (
|
||||
re.compile(r"^result-[a-f0-9]{64}$"),
|
||||
"missioncore.recorded-eomt-taxonomy/v1",
|
||||
16,
|
||||
"video/eomt-semantic-masks.zip",
|
||||
),
|
||||
"vegetation": (
|
||||
re.compile(r"^lab-v1-ravnoves-video-ddrnet-[a-f0-9]{64}$"),
|
||||
"missioncore.lab-v1-vegetation-taxonomy/v1",
|
||||
64,
|
||||
"video/ddrnet-semantic-masks.zip",
|
||||
),
|
||||
}[layer]
|
||||
result_id = _text(value.get("result_id"), f"{layer} result id")
|
||||
if expected[0].fullmatch(result_id) is None or value.get("frame_count") != 6830:
|
||||
raise CanonicalRecordedCatalogError(f"canonical {layer} layer identity changed")
|
||||
_text(value.get("name"), f"{layer} layer name")
|
||||
_finite_number(value.get("inference_fps"), f"{layer} inference FPS")
|
||||
_finite_number(value.get("latency_p95_ms"), f"{layer} latency")
|
||||
_nonnegative_integer(value.get("peak_reserved_vram_bytes"), f"{layer} VRAM")
|
||||
_validate_taxonomy(
|
||||
_object(value.get("taxonomy"), f"{layer} taxonomy"),
|
||||
schema=expected[1],
|
||||
class_count=expected[2],
|
||||
label=layer,
|
||||
)
|
||||
archive = _object(value.get("mask_archive"), f"{layer} mask archive")
|
||||
digest = _sha256_text(archive.get("sha256"), f"{layer} archive digest")
|
||||
byte_length = _positive_integer(archive.get("byte_length"), f"{layer} archive bytes")
|
||||
if archive.get("path") != expected[3]:
|
||||
raise CanonicalRecordedCatalogError(f"canonical {layer} archive changed")
|
||||
_require_artifact(
|
||||
artifacts,
|
||||
path=expected[3],
|
||||
sha256=digest,
|
||||
byte_length=byte_length,
|
||||
media_type="application/zip",
|
||||
)
|
||||
|
||||
|
||||
def _validate_taxonomy(
|
||||
value: dict[str, Any],
|
||||
*,
|
||||
schema: str,
|
||||
class_count: int,
|
||||
label: str,
|
||||
) -> None:
|
||||
classes = value.get("classes")
|
||||
if value.get("schema_version") != schema or not isinstance(classes, list):
|
||||
raise CanonicalRecordedCatalogError(f"canonical {label} taxonomy changed")
|
||||
if len(classes) != class_count:
|
||||
raise CanonicalRecordedCatalogError(f"canonical {label} taxonomy size changed")
|
||||
for expected_id, raw in enumerate(classes):
|
||||
item = _object(raw, f"{label} taxonomy class")
|
||||
color = item.get("color_rgb")
|
||||
if (
|
||||
item.get("class_id") != expected_id
|
||||
or isinstance(item.get("class_id"), bool)
|
||||
or not isinstance(color, list)
|
||||
or len(color) != 3
|
||||
or any(
|
||||
isinstance(channel, bool)
|
||||
or not isinstance(channel, int)
|
||||
or not 0 <= channel <= 255
|
||||
for channel in color
|
||||
)
|
||||
or item.get("disposition")
|
||||
not in {"labeled", "ambiguous", "prediction", "undefined"}
|
||||
):
|
||||
raise CanonicalRecordedCatalogError(f"canonical {label} taxonomy class changed")
|
||||
_text(item.get("label"), f"{label} taxonomy label")
|
||||
for optional in ("material_class", "evidence_state"):
|
||||
if item.get(optional) is not None:
|
||||
_text(item.get(optional), f"{label} taxonomy {optional}")
|
||||
|
||||
|
||||
def _artifact_catalog(value: object) -> dict[str, dict[str, Any]]:
|
||||
if not isinstance(value, list) or not value:
|
||||
raise CanonicalRecordedCatalogError("canonical result artifacts are missing")
|
||||
catalog: dict[str, dict[str, Any]] = {}
|
||||
for raw in value:
|
||||
descriptor = _object(raw, "canonical result artifact")
|
||||
path = _text(descriptor.get("path"), "canonical artifact path")
|
||||
if path in catalog:
|
||||
raise CanonicalRecordedCatalogError("canonical result artifact is duplicated")
|
||||
catalog[path] = descriptor
|
||||
return catalog
|
||||
|
||||
|
||||
def _require_artifact(
|
||||
artifacts: dict[str, dict[str, Any]],
|
||||
*,
|
||||
path: str,
|
||||
sha256: str,
|
||||
byte_length: int | None = None,
|
||||
media_type: str | None = None,
|
||||
) -> None:
|
||||
descriptor = artifacts.get(path)
|
||||
if (
|
||||
descriptor is None
|
||||
or descriptor.get("sha256") != sha256
|
||||
or (byte_length is not None and descriptor.get("byte_length") != byte_length)
|
||||
or (media_type is not None and descriptor.get("media_type") != media_type)
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical replay artifact binding changed")
|
||||
|
||||
|
||||
def _validate_timeline_payload(
|
||||
path: Path,
|
||||
*,
|
||||
expected_sha256: str,
|
||||
start_seconds: float,
|
||||
) -> None:
|
||||
try:
|
||||
payload = path.read_bytes()
|
||||
values = struct.unpack("<6830Q", payload)
|
||||
except (OSError, struct.error) as exc:
|
||||
raise CanonicalRecordedCatalogError("canonical timeline payload is invalid") from exc
|
||||
if (
|
||||
hashlib.sha256(payload).hexdigest() != expected_sha256
|
||||
or values[0] != round(start_seconds * 1_000_000_000)
|
||||
or values[-1] > 9_007_199_254_740_991
|
||||
or any(
|
||||
current <= previous
|
||||
for previous, current in zip(values, values[1:], strict=False)
|
||||
)
|
||||
):
|
||||
raise CanonicalRecordedCatalogError("canonical timeline payload changed")
|
||||
|
||||
|
||||
def _sha256_text(value: object, label: str) -> str:
|
||||
digest = _text(value, label)
|
||||
if re.fullmatch(r"[a-f0-9]{64}", digest) is None:
|
||||
raise CanonicalRecordedCatalogError(f"{label} is invalid")
|
||||
return digest
|
||||
|
||||
|
||||
def _nonnegative_integer(value: object, label: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_integer(value: object, label: str) -> int:
|
||||
integer = _nonnegative_integer(value, label)
|
||||
if integer == 0:
|
||||
raise CanonicalRecordedCatalogError(f"{label} must be positive")
|
||||
return integer
|
||||
@@ -20,6 +20,7 @@ from .media import (
|
||||
validate_recorded_media_timeline,
|
||||
)
|
||||
from .models import (
|
||||
LabReplayCapability,
|
||||
LabSessionBinding,
|
||||
LayoutConflictError,
|
||||
ObservationArtifactCandidate,
|
||||
@@ -57,6 +58,7 @@ from .store import (
|
||||
|
||||
__all__ = [
|
||||
"LayoutConflictError",
|
||||
"LabReplayCapability",
|
||||
"LabSessionBinding",
|
||||
"ActiveSessionLease",
|
||||
"ActiveSessionLeaseError",
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any, Literal
|
||||
|
||||
SessionStatus = Literal["ready", "interrupted", "failed"]
|
||||
SessionModality = Literal["point-cloud", "trajectory", "video"]
|
||||
LAB_REPLAY_CAPABILITY_SCHEMA = "missioncore.observation-lab-replay-capability/v1"
|
||||
|
||||
|
||||
class SessionStoreError(RuntimeError):
|
||||
@@ -28,6 +29,45 @@ class LayoutConflictError(SessionStoreError):
|
||||
"""A workspace layout revision changed since the caller loaded it."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LabReplayCapability:
|
||||
"""Explicit, observation-only admission for a derived recorded replay."""
|
||||
|
||||
schema_version: Literal["missioncore.observation-lab-replay-capability/v1"]
|
||||
kind: Literal["canonical-recorded-rerun"]
|
||||
viewer_profile: Literal["recorded-session"]
|
||||
timeline: Literal["session_time"]
|
||||
activation: Literal["explicit"]
|
||||
commands_enabled: Literal[False]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
expected_text = (
|
||||
LAB_REPLAY_CAPABILITY_SCHEMA,
|
||||
"canonical-recorded-rerun",
|
||||
"recorded-session",
|
||||
"session_time",
|
||||
"explicit",
|
||||
)
|
||||
if (
|
||||
self.schema_version,
|
||||
self.kind,
|
||||
self.viewer_profile,
|
||||
self.timeline,
|
||||
self.activation,
|
||||
) != expected_text or self.commands_enabled is not False:
|
||||
raise ValueError("LAB replay capability is invalid")
|
||||
|
||||
def as_dict(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": self.schema_version,
|
||||
"kind": self.kind,
|
||||
"viewer_profile": self.viewer_profile,
|
||||
"timeline": self.timeline,
|
||||
"activation": self.activation,
|
||||
"commands_enabled": self.commands_enabled,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LabSessionBinding:
|
||||
"""Immutable provenance for one derived laboratory replay."""
|
||||
@@ -41,10 +81,15 @@ class LabSessionBinding:
|
||||
config_sha256: str | None
|
||||
run_created_at_utc: str
|
||||
published_at_utc: str
|
||||
replay_capability: LabReplayCapability | None
|
||||
provenance: dict[str, Any]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
def as_dict(
|
||||
self,
|
||||
*,
|
||||
include_replay_capability: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
document: dict[str, Any] = {
|
||||
"lab_id": self.lab_id,
|
||||
"source_session_id": self.source_session_id,
|
||||
"result_kind": self.result_kind,
|
||||
@@ -55,6 +100,11 @@ class LabSessionBinding:
|
||||
"published_at_utc": self.published_at_utc,
|
||||
"provenance": self.provenance,
|
||||
}
|
||||
if include_replay_capability:
|
||||
document["replay_capability"] = (
|
||||
None if self.replay_capability is None else self.replay_capability.as_dict()
|
||||
)
|
||||
return document
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -137,6 +187,8 @@ class SessionDetail:
|
||||
summary: SessionSummary
|
||||
sources: tuple[SessionSource, ...]
|
||||
artifacts: tuple[SessionArtifact, ...]
|
||||
plugin_id: str
|
||||
archive_id: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
duration = self.summary.duration_seconds
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
@@ -17,6 +18,7 @@ from uuid import uuid4
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
from .models import (
|
||||
LabReplayCapability,
|
||||
LabSessionBinding,
|
||||
LayoutConflictError,
|
||||
ObservationArtifactCandidate,
|
||||
@@ -114,6 +116,8 @@ CREATE TABLE IF NOT EXISTS observation_lab_instances (
|
||||
config_sha256 TEXT,
|
||||
run_created_at_utc TEXT NOT NULL,
|
||||
published_at_utc TEXT NOT NULL,
|
||||
include_recorded_media INTEGER CHECK (include_recorded_media IN (0, 1)),
|
||||
replay_capability_json TEXT,
|
||||
provenance_json TEXT NOT NULL
|
||||
);
|
||||
|
||||
@@ -213,10 +217,13 @@ class SessionStore:
|
||||
limit: int = 20,
|
||||
cursor: str | None = None,
|
||||
scope: SessionScope = "all",
|
||||
include_capability_projections: bool = True,
|
||||
) -> SessionPage:
|
||||
if not 1 <= limit <= 100:
|
||||
raise ValueError("limit must be within 1..100")
|
||||
scope_clause = {
|
||||
if not isinstance(include_capability_projections, bool):
|
||||
raise ValueError("capability projection policy must be boolean")
|
||||
scope_clause = ({
|
||||
"all": "1 = 1",
|
||||
"source": (
|
||||
"NOT EXISTS (SELECT 1 FROM observation_lab_instances AS lab "
|
||||
@@ -226,7 +233,22 @@ class SessionStore:
|
||||
"EXISTS (SELECT 1 FROM observation_lab_instances AS lab "
|
||||
"WHERE lab.session_id = sessions.session_id)"
|
||||
),
|
||||
}.get(scope)
|
||||
} if include_capability_projections else {
|
||||
"all": (
|
||||
"NOT EXISTS (SELECT 1 FROM observation_lab_instances AS lab "
|
||||
"WHERE lab.session_id = sessions.session_id "
|
||||
"AND lab.replay_capability_json IS NOT NULL)"
|
||||
),
|
||||
"source": (
|
||||
"NOT EXISTS (SELECT 1 FROM observation_lab_instances AS lab "
|
||||
"WHERE lab.session_id = sessions.session_id)"
|
||||
),
|
||||
"laboratory": (
|
||||
"EXISTS (SELECT 1 FROM observation_lab_instances AS lab "
|
||||
"WHERE lab.session_id = sessions.session_id "
|
||||
"AND lab.replay_capability_json IS NULL)"
|
||||
),
|
||||
}).get(scope)
|
||||
if scope_clause is None:
|
||||
raise ValueError("scope must be all, source, or laboratory")
|
||||
parameters: list[object] = []
|
||||
@@ -283,8 +305,18 @@ class SessionStore:
|
||||
return SessionPage(items=items, next_cursor=next_cursor)
|
||||
|
||||
def get_session(self, session_id: str) -> SessionDetail:
|
||||
detail, _snapshot_sha256 = self.get_session_with_catalog_snapshot(session_id)
|
||||
return detail
|
||||
|
||||
def get_session_with_catalog_snapshot(
|
||||
self,
|
||||
session_id: str,
|
||||
) -> tuple[SessionDetail, str]:
|
||||
"""Read one detail and its private catalog snapshot from one SQLite view."""
|
||||
|
||||
_validate_identifier(session_id, "session id")
|
||||
with self._connect() as connection:
|
||||
connection.execute("BEGIN")
|
||||
row = connection.execute(
|
||||
"SELECT * FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
@@ -292,19 +324,20 @@ class SessionStore:
|
||||
if row is None:
|
||||
raise SessionNotFoundError("observation session was not found")
|
||||
source_rows = connection.execute(
|
||||
"SELECT source_id, semantic_channel_id, modality, status, seekable, artifact_id "
|
||||
"FROM observation_session_sources WHERE session_id = ? ORDER BY source_id",
|
||||
"SELECT * FROM observation_session_sources "
|
||||
"WHERE session_id = ? ORDER BY source_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
artifact_rows = connection.execute(
|
||||
"SELECT artifact_id, kind, media_type, byte_length, sha256, integrity_status "
|
||||
"FROM observation_session_artifacts WHERE session_id = ? ORDER BY artifact_id",
|
||||
"SELECT * FROM observation_session_artifacts "
|
||||
"WHERE session_id = ? ORDER BY artifact_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
lab_row = connection.execute(
|
||||
"SELECT * FROM observation_lab_instances WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
snapshot_sha256 = _catalog_snapshot_sha256(row, source_rows, artifact_rows)
|
||||
sources = tuple(
|
||||
SessionSource(
|
||||
source_id=source["source_id"],
|
||||
@@ -327,14 +360,17 @@ class SessionStore:
|
||||
)
|
||||
for artifact in artifact_rows
|
||||
)
|
||||
return SessionDetail(
|
||||
detail = SessionDetail(
|
||||
summary=_summary_from_row(
|
||||
row,
|
||||
lab=None if lab_row is None else _lab_binding_from_row(lab_row),
|
||||
),
|
||||
sources=sources,
|
||||
artifacts=artifacts,
|
||||
plugin_id=row["plugin_id"],
|
||||
archive_id=row["archive_id"],
|
||||
)
|
||||
return detail, snapshot_sha256
|
||||
|
||||
def get_lab_instance(self, session_id: str) -> LabSessionBinding | None:
|
||||
"""Return immutable LAB provenance without exposing filesystem locators."""
|
||||
@@ -359,9 +395,11 @@ class SessionStore:
|
||||
run_created_at_utc: str,
|
||||
source_result_id: str | None = None,
|
||||
config_sha256: str | None = None,
|
||||
replay_capability: LabReplayCapability | None = None,
|
||||
provenance: dict[str, Any] | None = None,
|
||||
duration_seconds: float | None = None,
|
||||
include_recorded_media: bool = True,
|
||||
expected_source_catalog_sha256: str | None = None,
|
||||
) -> LabSessionBinding:
|
||||
"""Append one immutable catalog projection over an existing source session.
|
||||
|
||||
@@ -392,9 +430,22 @@ class SessionStore:
|
||||
or duration_seconds <= 0
|
||||
):
|
||||
raise ValueError("LAB duration must be a positive finite value")
|
||||
if not isinstance(include_recorded_media, bool):
|
||||
raise ValueError("LAB recorded-media policy must be boolean")
|
||||
if (
|
||||
expected_source_catalog_sha256 is not None
|
||||
and SHA256_PATTERN.fullmatch(expected_source_catalog_sha256) is None
|
||||
):
|
||||
raise ValueError("LAB source catalog snapshot SHA-256 is invalid")
|
||||
normalized_provenance = provenance or {}
|
||||
_validate_lab_method(normalized_provenance)
|
||||
_validate_replay_capability_provenance(
|
||||
normalized_provenance,
|
||||
replay_capability,
|
||||
)
|
||||
serialized_provenance = _serialize_provenance(normalized_provenance)
|
||||
serialized_replay_capability = _serialize_replay_capability(replay_capability)
|
||||
serialized_recorded_media_policy = int(include_recorded_media)
|
||||
published_at = utc_now_iso()
|
||||
|
||||
with self._lock, self._connect() as connection:
|
||||
@@ -406,6 +457,19 @@ class SessionStore:
|
||||
if source is None:
|
||||
connection.rollback()
|
||||
raise SessionNotFoundError("LAB source observation session was not found")
|
||||
if (
|
||||
expected_source_catalog_sha256 is not None
|
||||
and _catalog_snapshot_sha256_for_session(
|
||||
connection,
|
||||
source_session_id,
|
||||
session_row=source,
|
||||
)
|
||||
!= expected_source_catalog_sha256
|
||||
):
|
||||
connection.rollback()
|
||||
raise SessionIntegrityError(
|
||||
"LAB source catalog changed after admission"
|
||||
)
|
||||
if (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM observation_lab_instances WHERE session_id = ?",
|
||||
@@ -428,6 +492,7 @@ class SessionStore:
|
||||
"source_result_id": source_result_id,
|
||||
"config_sha256": config_sha256,
|
||||
"run_created_at_utc": run_created_at_utc,
|
||||
"replay_capability_json": serialized_replay_capability,
|
||||
"provenance_json": serialized_provenance,
|
||||
}
|
||||
if existing is not None:
|
||||
@@ -436,6 +501,36 @@ class SessionStore:
|
||||
raise SessionIntegrityError(
|
||||
"LAB session id is already bound to different provenance"
|
||||
)
|
||||
if replay_capability is not None:
|
||||
stored_recorded_media_policy = existing["include_recorded_media"]
|
||||
if (
|
||||
stored_recorded_media_policy is not None
|
||||
and stored_recorded_media_policy != serialized_recorded_media_policy
|
||||
):
|
||||
connection.rollback()
|
||||
raise SessionIntegrityError(
|
||||
"LAB session id is already bound to a different recorded-media policy"
|
||||
)
|
||||
_validate_existing_lab_projection(
|
||||
connection,
|
||||
session_id=session_id,
|
||||
source_session_id=source_session_id,
|
||||
display_name=normalized_name,
|
||||
run_created_at_utc=run_created_at_utc,
|
||||
duration_seconds=(
|
||||
source["duration_seconds"]
|
||||
if duration_seconds is None
|
||||
else duration_seconds
|
||||
),
|
||||
include_recorded_media=include_recorded_media,
|
||||
)
|
||||
if stored_recorded_media_policy is None:
|
||||
connection.execute(
|
||||
"UPDATE observation_lab_instances SET include_recorded_media = ? "
|
||||
"WHERE session_id = ?",
|
||||
(serialized_recorded_media_policy, session_id),
|
||||
)
|
||||
_synchronize_lab_projection_summary(connection, session_id)
|
||||
connection.commit()
|
||||
return _lab_binding_from_row(existing)
|
||||
if (
|
||||
@@ -507,8 +602,8 @@ class SessionStore:
|
||||
"INSERT INTO observation_lab_instances "
|
||||
"(session_id, source_session_id, lab_id, result_kind, result_id, "
|
||||
"source_result_id, config_sha256, run_created_at_utc, "
|
||||
"published_at_utc, provenance_json) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
"published_at_utc, include_recorded_media, replay_capability_json, "
|
||||
"provenance_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
session_id,
|
||||
source_session_id,
|
||||
@@ -519,9 +614,13 @@ class SessionStore:
|
||||
config_sha256,
|
||||
run_created_at_utc,
|
||||
published_at,
|
||||
serialized_recorded_media_policy,
|
||||
serialized_replay_capability,
|
||||
serialized_provenance,
|
||||
),
|
||||
)
|
||||
if replay_capability is not None:
|
||||
_synchronize_lab_projection_summary(connection, session_id)
|
||||
connection.commit()
|
||||
binding = self.get_lab_instance(session_id)
|
||||
if binding is None:
|
||||
@@ -817,6 +916,22 @@ class SessionStore:
|
||||
"ALTER TABLE observation_session_artifacts "
|
||||
"ADD COLUMN replay_byte_length INTEGER NOT NULL DEFAULT 0"
|
||||
)
|
||||
lab_columns = {
|
||||
row["name"]
|
||||
for row in connection.execute("PRAGMA table_info(observation_lab_instances)")
|
||||
}
|
||||
if "replay_capability_json" not in lab_columns:
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_lab_instances "
|
||||
"ADD COLUMN replay_capability_json TEXT"
|
||||
)
|
||||
if "include_recorded_media" not in lab_columns:
|
||||
connection.execute(
|
||||
"ALTER TABLE observation_lab_instances "
|
||||
"ADD COLUMN include_recorded_media INTEGER "
|
||||
"CHECK (include_recorded_media IN (0, 1))"
|
||||
)
|
||||
_migrate_canonical_replay_capabilities(connection)
|
||||
connection.commit()
|
||||
with _ignore_os_error():
|
||||
self.database_path.chmod(0o600)
|
||||
@@ -1046,6 +1161,428 @@ def _validate_candidate_replay(
|
||||
raise SessionIntegrityError("non-replayable observation declares replay artifacts")
|
||||
|
||||
|
||||
def _validate_existing_lab_projection(
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
session_id: str,
|
||||
source_session_id: str,
|
||||
display_name: str,
|
||||
run_created_at_utc: str,
|
||||
duration_seconds: float | None,
|
||||
include_recorded_media: bool,
|
||||
) -> None:
|
||||
"""Validate the immutable companion snapshot without rereading a mutable source row."""
|
||||
|
||||
summary = connection.execute(
|
||||
"SELECT * FROM observation_sessions "
|
||||
"WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
source_summary = connection.execute(
|
||||
"SELECT * FROM observation_sessions WHERE session_id = ?",
|
||||
(source_session_id,),
|
||||
).fetchone()
|
||||
if (
|
||||
summary is None
|
||||
or source_summary is None
|
||||
or summary["display_name"] != display_name
|
||||
or summary["duration_seconds"] != duration_seconds
|
||||
or summary["plugin_id"] != source_summary["plugin_id"]
|
||||
or summary["archive_id"] != LAB_ARCHIVE_ID
|
||||
or summary["status"] != source_summary["status"]
|
||||
or summary["started_at_utc"] != run_created_at_utc
|
||||
or summary["completed_at_utc"] != run_created_at_utc
|
||||
or summary["replayable"] != source_summary["replayable"]
|
||||
or summary["origin"] != LAB_ORIGIN
|
||||
or summary["primary_replay_artifact_id"]
|
||||
!= source_summary["primary_replay_artifact_id"]
|
||||
or summary["timeline_origin_epoch_ns"]
|
||||
!= source_summary["timeline_origin_epoch_ns"]
|
||||
or summary["timeline_origin_monotonic_ns"]
|
||||
!= source_summary["timeline_origin_monotonic_ns"]
|
||||
or summary["allowed_root"] != source_summary["allowed_root"]
|
||||
or summary["session_root"] != source_summary["session_root"]
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"LAB session id is already bound to different catalog metadata"
|
||||
)
|
||||
if (
|
||||
summary["status"] != "ready"
|
||||
or summary["replayable"] != 1
|
||||
or summary["primary_replay_artifact_id"] is None
|
||||
or summary["timeline_origin_epoch_ns"] is None
|
||||
or summary["timeline_origin_monotonic_ns"] is None
|
||||
or summary["timeline_origin_epoch_ns"] < 0
|
||||
or summary["timeline_origin_monotonic_ns"] < 0
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"LAB replay projection is not a complete ready recording"
|
||||
)
|
||||
if not include_recorded_media and (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM observation_session_sources "
|
||||
"WHERE session_id = ? AND modality = 'video' LIMIT 1",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
is not None
|
||||
or connection.execute(
|
||||
"SELECT 1 FROM observation_session_artifacts "
|
||||
"WHERE session_id = ? AND kind = 'recorded-video' LIMIT 1",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
is not None
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"LAB session id is already bound to a different recorded-media policy"
|
||||
)
|
||||
_validate_lab_projection_rows_match_source(
|
||||
connection,
|
||||
session_id=session_id,
|
||||
source_session_id=source_session_id,
|
||||
include_recorded_media=include_recorded_media,
|
||||
)
|
||||
|
||||
|
||||
def _validate_lab_projection_rows_match_source(
|
||||
connection: sqlite3.Connection,
|
||||
*,
|
||||
session_id: str,
|
||||
source_session_id: str,
|
||||
include_recorded_media: bool,
|
||||
) -> None:
|
||||
source_filter = "" if include_recorded_media else " AND modality <> 'video'"
|
||||
artifact_filter = "" if include_recorded_media else " AND kind <> 'recorded-video'"
|
||||
source_columns = (
|
||||
"source_id",
|
||||
"semantic_channel_id",
|
||||
"modality",
|
||||
"status",
|
||||
"seekable",
|
||||
"artifact_id",
|
||||
)
|
||||
artifact_columns = (
|
||||
"artifact_id",
|
||||
"kind",
|
||||
"media_type",
|
||||
"byte_length",
|
||||
"sha256",
|
||||
"integrity_status",
|
||||
"locator",
|
||||
"replay_byte_length",
|
||||
)
|
||||
expected_sources = connection.execute(
|
||||
"SELECT * FROM observation_session_sources WHERE session_id = ?"
|
||||
f"{source_filter} ORDER BY source_id", # noqa: S608 - closed static fragment
|
||||
(source_session_id,),
|
||||
).fetchall()
|
||||
actual_sources = connection.execute(
|
||||
"SELECT * FROM observation_session_sources "
|
||||
"WHERE session_id = ? ORDER BY source_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
expected_artifacts = connection.execute(
|
||||
"SELECT * FROM observation_session_artifacts WHERE session_id = ?"
|
||||
f"{artifact_filter} ORDER BY artifact_id", # noqa: S608 - closed static fragment
|
||||
(source_session_id,),
|
||||
).fetchall()
|
||||
actual_artifacts = connection.execute(
|
||||
"SELECT * FROM observation_session_artifacts "
|
||||
"WHERE session_id = ? ORDER BY artifact_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
|
||||
def values(
|
||||
rows: list[sqlite3.Row],
|
||||
columns: tuple[str, ...],
|
||||
) -> tuple[tuple[object, ...], ...]:
|
||||
return tuple(tuple(row[column] for column in columns) for row in rows)
|
||||
|
||||
if (
|
||||
values(actual_sources, source_columns)
|
||||
!= values(expected_sources, source_columns)
|
||||
or values(actual_artifacts, artifact_columns)
|
||||
!= values(expected_artifacts, artifact_columns)
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"LAB replay projection no longer matches its admitted source snapshot"
|
||||
)
|
||||
|
||||
|
||||
def _synchronize_lab_projection_summary(
|
||||
connection: sqlite3.Connection,
|
||||
session_id: str,
|
||||
) -> None:
|
||||
"""Make the synthetic summary describe exactly its copied rows."""
|
||||
|
||||
summary = connection.execute(
|
||||
"SELECT modalities_json, source_count, total_bytes, replayable, "
|
||||
"primary_replay_artifact_id "
|
||||
"FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
if summary is None:
|
||||
raise SessionIntegrityError("LAB projection catalog row is missing")
|
||||
try:
|
||||
declared_modalities = json.loads(summary["modalities_json"])
|
||||
except (TypeError, json.JSONDecodeError) as exc:
|
||||
raise SessionIntegrityError("LAB projection modalities are invalid") from exc
|
||||
if not isinstance(declared_modalities, list) or not all(
|
||||
isinstance(value, str) for value in declared_modalities
|
||||
):
|
||||
raise SessionIntegrityError("LAB projection modalities are invalid")
|
||||
|
||||
actual_modalities = {
|
||||
row["modality"]
|
||||
for row in connection.execute(
|
||||
"SELECT DISTINCT modality FROM observation_session_sources "
|
||||
"WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
}
|
||||
modalities: list[str] = []
|
||||
for value in declared_modalities:
|
||||
if value in actual_modalities and value not in modalities:
|
||||
modalities.append(value)
|
||||
modalities.extend(sorted(actual_modalities.difference(modalities)))
|
||||
source_count = connection.execute(
|
||||
"SELECT COUNT(*) AS value FROM observation_session_sources WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()["value"]
|
||||
total_bytes = connection.execute(
|
||||
"SELECT COALESCE(SUM(byte_length), 0) AS value "
|
||||
"FROM observation_session_artifacts WHERE session_id = ? "
|
||||
"AND artifact_id IN ("
|
||||
"SELECT DISTINCT artifact_id FROM observation_session_sources "
|
||||
"WHERE session_id = ?)",
|
||||
(session_id, session_id),
|
||||
).fetchone()["value"]
|
||||
primary_artifact_id = summary["primary_replay_artifact_id"]
|
||||
if summary["replayable"] and (
|
||||
primary_artifact_id is None
|
||||
or connection.execute(
|
||||
"SELECT 1 FROM observation_session_artifacts "
|
||||
"WHERE session_id = ? AND artifact_id = ?",
|
||||
(session_id, primary_artifact_id),
|
||||
).fetchone()
|
||||
is None
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"LAB projection omitted its primary replay artifact"
|
||||
)
|
||||
modalities_json = json.dumps(modalities, separators=(",", ":"))
|
||||
if (
|
||||
summary["modalities_json"] != modalities_json
|
||||
or summary["source_count"] != source_count
|
||||
or summary["total_bytes"] != total_bytes
|
||||
):
|
||||
connection.execute(
|
||||
"UPDATE observation_sessions SET modalities_json = ?, source_count = ?, "
|
||||
"total_bytes = ?, updated_at_utc = ? WHERE session_id = ?",
|
||||
(
|
||||
modalities_json,
|
||||
source_count,
|
||||
total_bytes,
|
||||
utc_now_iso(),
|
||||
session_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _catalog_snapshot_sha256_for_session(
|
||||
connection: sqlite3.Connection,
|
||||
session_id: str,
|
||||
*,
|
||||
session_row: sqlite3.Row | None = None,
|
||||
) -> str:
|
||||
row = (
|
||||
session_row
|
||||
if session_row is not None
|
||||
else connection.execute(
|
||||
"SELECT * FROM observation_sessions WHERE session_id = ?",
|
||||
(session_id,),
|
||||
).fetchone()
|
||||
)
|
||||
if row is None:
|
||||
raise SessionNotFoundError("observation session was not found")
|
||||
source_rows = connection.execute(
|
||||
"SELECT * FROM observation_session_sources "
|
||||
"WHERE session_id = ? ORDER BY source_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
artifact_rows = connection.execute(
|
||||
"SELECT * FROM observation_session_artifacts "
|
||||
"WHERE session_id = ? ORDER BY artifact_id",
|
||||
(session_id,),
|
||||
).fetchall()
|
||||
return _catalog_snapshot_sha256(row, source_rows, artifact_rows)
|
||||
|
||||
|
||||
def _catalog_snapshot_sha256(
|
||||
session_row: sqlite3.Row,
|
||||
source_rows: list[sqlite3.Row],
|
||||
artifact_rows: list[sqlite3.Row],
|
||||
) -> str:
|
||||
"""Bind every stored field copied or referenced by a LAB projection."""
|
||||
|
||||
payload = {
|
||||
"session": dict(session_row),
|
||||
"sources": [dict(row) for row in source_rows],
|
||||
"artifacts": [dict(row) for row in artifact_rows],
|
||||
}
|
||||
serialized = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(serialized).hexdigest()
|
||||
|
||||
|
||||
def _migrate_canonical_replay_capabilities(connection: sqlite3.Connection) -> None:
|
||||
"""Type the one rolling-upgrade projection that predates the v2 column."""
|
||||
|
||||
rows = connection.execute(
|
||||
"SELECT * FROM observation_lab_instances "
|
||||
"WHERE replay_capability_json IS NULL OR include_recorded_media IS NULL"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
provenance = json.loads(row["provenance_json"])
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
continue
|
||||
capability = _canonical_rolling_capability(row, provenance)
|
||||
if capability is None:
|
||||
continue
|
||||
serialized = _serialize_replay_capability(capability)
|
||||
stored = row["replay_capability_json"]
|
||||
if stored is not None and stored != serialized:
|
||||
raise SessionIntegrityError(
|
||||
"stored canonical LAB replay capability conflicts with provenance"
|
||||
)
|
||||
summary = connection.execute(
|
||||
"SELECT * FROM observation_sessions WHERE session_id = ?",
|
||||
(row["session_id"],),
|
||||
).fetchone()
|
||||
if summary is None:
|
||||
raise SessionIntegrityError("canonical LAB catalog row is missing")
|
||||
source_summary = connection.execute(
|
||||
"SELECT * FROM observation_sessions WHERE session_id = ?",
|
||||
(row["source_session_id"],),
|
||||
).fetchone()
|
||||
if (
|
||||
summary["display_name"]
|
||||
!= "RAVNOVES004TREE · полный маршрут восприятия"
|
||||
or summary["duration_seconds"] != 718.0
|
||||
or row["run_created_at_utc"]
|
||||
!= "2026-08-29T18:05:11.329061+00:00"
|
||||
or source_summary is None
|
||||
or source_summary["plugin_id"]
|
||||
!= "nodedc.device.xgrids-lixelkity-k1"
|
||||
or source_summary["archive_id"] != "xgrids-k1.viewer-live.evidence"
|
||||
):
|
||||
raise SessionIntegrityError(
|
||||
"canonical rolling LAB projection metadata is invalid"
|
||||
)
|
||||
_validate_existing_lab_projection(
|
||||
connection,
|
||||
session_id=row["session_id"],
|
||||
source_session_id=row["source_session_id"],
|
||||
display_name=summary["display_name"],
|
||||
run_created_at_utc=row["run_created_at_utc"],
|
||||
duration_seconds=summary["duration_seconds"],
|
||||
include_recorded_media=False,
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE observation_lab_instances SET replay_capability_json = ?, "
|
||||
"include_recorded_media = 0 WHERE session_id = ?",
|
||||
(serialized, row["session_id"]),
|
||||
)
|
||||
_synchronize_lab_projection_summary(connection, row["session_id"])
|
||||
|
||||
|
||||
def _canonical_rolling_capability(
|
||||
row: sqlite3.Row,
|
||||
provenance: object,
|
||||
) -> LabReplayCapability | None:
|
||||
if not isinstance(provenance, dict) or set(provenance) != {
|
||||
"schema_version",
|
||||
"evidence_identity_sha256",
|
||||
"result_document_sha256",
|
||||
"replay_capability",
|
||||
"authority",
|
||||
"method",
|
||||
}:
|
||||
return None
|
||||
evidence_identity = provenance.get("evidence_identity_sha256")
|
||||
result_document = provenance.get("result_document_sha256")
|
||||
expected_result_id = (
|
||||
f"lab-v1-vegetation-shadow-{evidence_identity}"
|
||||
if isinstance(evidence_identity, str)
|
||||
else None
|
||||
)
|
||||
if (
|
||||
provenance.get("schema_version")
|
||||
!= "missioncore.canonical-recorded-lab-projection/v1"
|
||||
or not isinstance(evidence_identity, str)
|
||||
or SHA256_PATTERN.fullmatch(evidence_identity) is None
|
||||
or not isinstance(result_document, str)
|
||||
or SHA256_PATTERN.fullmatch(result_document) is None
|
||||
or row["session_id"] != expected_result_id
|
||||
or row["result_id"] != expected_result_id
|
||||
or row["source_session_id"] != "20260828T130511Z_viewer_live"
|
||||
or row["lab_id"] != "LAB V1"
|
||||
or row["result_kind"] != "recorded-perception-qualification"
|
||||
or row["config_sha256"] is not None
|
||||
or not isinstance(row["source_result_id"], str)
|
||||
or re.fullmatch(
|
||||
r"lab-v1-vegetation-shadow-[a-f0-9]{64}",
|
||||
row["source_result_id"],
|
||||
)
|
||||
is None
|
||||
):
|
||||
return None
|
||||
authority = provenance.get("authority")
|
||||
authority_keys = {
|
||||
"commands_enabled",
|
||||
"navigation_or_safety_accepted",
|
||||
"actuation_accepted",
|
||||
}
|
||||
if (
|
||||
not isinstance(authority, dict)
|
||||
or set(authority) != authority_keys
|
||||
or any(authority[key] is not False for key in authority_keys)
|
||||
):
|
||||
return None
|
||||
method = provenance.get("method")
|
||||
expected_component = {
|
||||
"kind": "source",
|
||||
"name": "sealed full-route LAB result",
|
||||
"version": "missioncore.lab-v1-vegetation-shadow/v1",
|
||||
"role": "immutable Session catalog projection",
|
||||
"identity_sha256": evidence_identity,
|
||||
}
|
||||
if method != {
|
||||
"schema_version": LAB_METHOD_SCHEMA,
|
||||
"completeness": "legacy-partial",
|
||||
"execution_class": "ai-inference",
|
||||
"pipeline_id": "ravnoves004tree-full-eomt-ddrnet-recorded-review/v1",
|
||||
"components": [expected_component],
|
||||
}:
|
||||
return None
|
||||
capability_document = provenance.get("replay_capability")
|
||||
try:
|
||||
serialized = json.dumps(
|
||||
capability_document,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
return _replay_capability_from_row(serialized)
|
||||
except (TypeError, ValueError, SessionIntegrityError):
|
||||
return None
|
||||
|
||||
|
||||
def _summary_from_row(
|
||||
row: sqlite3.Row,
|
||||
*,
|
||||
@@ -1076,6 +1613,13 @@ def _lab_binding_from_row(row: sqlite3.Row) -> LabSessionBinding:
|
||||
raise SessionIntegrityError("stored LAB provenance is invalid") from exc
|
||||
if not isinstance(provenance, dict):
|
||||
raise SessionIntegrityError("stored LAB provenance is not an object")
|
||||
replay_capability = _replay_capability_from_row(row["replay_capability_json"])
|
||||
try:
|
||||
_validate_replay_capability_provenance(provenance, replay_capability)
|
||||
except ValueError as exc:
|
||||
raise SessionIntegrityError(
|
||||
"stored LAB replay capability does not match provenance"
|
||||
) from exc
|
||||
return LabSessionBinding(
|
||||
session_id=row["session_id"],
|
||||
source_session_id=row["source_session_id"],
|
||||
@@ -1086,10 +1630,66 @@ def _lab_binding_from_row(row: sqlite3.Row) -> LabSessionBinding:
|
||||
config_sha256=row["config_sha256"],
|
||||
run_created_at_utc=row["run_created_at_utc"],
|
||||
published_at_utc=row["published_at_utc"],
|
||||
replay_capability=replay_capability,
|
||||
provenance=provenance,
|
||||
)
|
||||
|
||||
|
||||
def _serialize_replay_capability(value: LabReplayCapability | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, LabReplayCapability):
|
||||
raise ValueError("LAB replay capability must use the typed contract")
|
||||
return json.dumps(value.as_dict(), sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _replay_capability_from_row(value: object) -> LabReplayCapability | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
document = json.loads(value)
|
||||
except (TypeError, json.JSONDecodeError) as exc:
|
||||
raise SessionIntegrityError("stored LAB replay capability is invalid") from exc
|
||||
expected_keys = {
|
||||
"schema_version",
|
||||
"kind",
|
||||
"viewer_profile",
|
||||
"timeline",
|
||||
"activation",
|
||||
"commands_enabled",
|
||||
}
|
||||
if not isinstance(document, dict) or set(document) != expected_keys:
|
||||
raise SessionIntegrityError("stored LAB replay capability is invalid")
|
||||
try:
|
||||
return LabReplayCapability(
|
||||
schema_version=document["schema_version"],
|
||||
kind=document["kind"],
|
||||
viewer_profile=document["viewer_profile"],
|
||||
timeline=document["timeline"],
|
||||
activation=document["activation"],
|
||||
commands_enabled=document["commands_enabled"],
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise SessionIntegrityError("stored LAB replay capability is invalid") from exc
|
||||
|
||||
|
||||
def _validate_replay_capability_provenance(
|
||||
provenance: dict[str, Any],
|
||||
capability: LabReplayCapability | None,
|
||||
) -> None:
|
||||
declared = provenance.get("replay_capability")
|
||||
if capability is None:
|
||||
if declared is not None:
|
||||
raise ValueError("LAB provenance cannot grant an untyped replay capability")
|
||||
return
|
||||
if (
|
||||
not isinstance(declared, dict)
|
||||
or _serialize_provenance(declared)
|
||||
!= _serialize_provenance(capability.as_dict())
|
||||
):
|
||||
raise ValueError("LAB replay capability must exactly match provenance")
|
||||
|
||||
|
||||
def _serialize_provenance(value: dict[str, Any]) -> str:
|
||||
try:
|
||||
serialized = json.dumps(
|
||||
|
||||
@@ -386,7 +386,11 @@ def finalized_replayable_recording_ids() -> tuple[str, ...]:
|
||||
finalized: list[str] = []
|
||||
cursor: str | None = None
|
||||
while True:
|
||||
page = session_store.list_recent(limit=100, cursor=cursor)
|
||||
page = session_store.list_recent(
|
||||
limit=100,
|
||||
cursor=cursor,
|
||||
include_capability_projections=False,
|
||||
)
|
||||
finalized.extend(
|
||||
summary.session_id
|
||||
for summary in page.items
|
||||
|
||||
@@ -341,10 +341,16 @@ def build_session_router(
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, max_length=128),
|
||||
scope: Literal["all", "source", "laboratory"] = "all",
|
||||
lab_contract: Literal["v1", "v2"] = "v1",
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
_refresh_catalog(catalog_refresher)
|
||||
page = store.list_recent(limit=limit, cursor=cursor, scope=scope)
|
||||
page = store.list_recent(
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
scope=scope,
|
||||
include_capability_projections=lab_contract == "v2",
|
||||
)
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
@@ -356,7 +362,15 @@ def build_session_router(
|
||||
"modalities": list(item.modalities),
|
||||
"duration_seconds": item.duration_seconds or 0.0,
|
||||
"replayable": item.replayable,
|
||||
**({"lab": item.lab.as_dict()} if item.lab is not None else {}),
|
||||
**(
|
||||
{
|
||||
"lab": item.lab.as_dict(
|
||||
include_replay_capability=lab_contract == "v2"
|
||||
)
|
||||
}
|
||||
if item.lab is not None
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{
|
||||
"preparation": _catalog_preparation_document(
|
||||
|
||||
Reference in New Issue
Block a user