feat(lab): publish M4.8 assisted regression evidence
This commit is contained in:
DCCONSTRUCTIONS
@@ -2,8 +2,10 @@
|
||||
|
||||
from k1link.laboratory.evidence_registry import (
|
||||
LABORATORY_EVIDENCE_DEFINITION_SCHEMA,
|
||||
LABORATORY_EVIDENCE_LIFECYCLE_DEFINITION_SCHEMA,
|
||||
LaboratoryEvidenceDefinition,
|
||||
LaboratoryEvidenceRegistry,
|
||||
LaboratoryEvidenceVariant,
|
||||
LaboratoryRegistryError,
|
||||
)
|
||||
from k1link.laboratory.evidence_report import (
|
||||
@@ -34,9 +36,11 @@ from k1link.laboratory.value_review_registry import (
|
||||
|
||||
__all__ = [
|
||||
"LABORATORY_EVIDENCE_DEFINITION_SCHEMA",
|
||||
"LABORATORY_EVIDENCE_LIFECYCLE_DEFINITION_SCHEMA",
|
||||
"LABORATORY_EVIDENCE_REPORT_SCHEMA",
|
||||
"LaboratoryEvidenceDefinition",
|
||||
"LaboratoryEvidenceRegistry",
|
||||
"LaboratoryEvidenceVariant",
|
||||
"LaboratoryEvidenceReportError",
|
||||
"LaboratoryEvidenceReportNotFound",
|
||||
"LaboratoryEvidenceReportService",
|
||||
|
||||
@@ -7,13 +7,22 @@ from pathlib import Path, PurePosixPath
|
||||
from typing import Final
|
||||
|
||||
LABORATORY_EVIDENCE_DEFINITION_SCHEMA: Final = "missioncore.laboratory-evidence-definition/v1"
|
||||
LABORATORY_EVIDENCE_LIFECYCLE_DEFINITION_SCHEMA: Final = (
|
||||
"missioncore.laboratory-evidence-definition/v2"
|
||||
)
|
||||
_DEFINITION_MAX_BYTES: Final = 16 * 1024
|
||||
_IDENTIFIER = re.compile(r"^[a-z][a-z0-9-]{2,95}$")
|
||||
_SCHEMA_VERSION = re.compile(r"^missioncore\.[a-z0-9.-]+/v[1-9][0-9]*$")
|
||||
_TOP_LEVEL_KEYS: Final = frozenset({"schema_version", "work_id", "evidence"})
|
||||
_LIFECYCLE_TOP_LEVEL_KEYS: Final = frozenset(
|
||||
{"schema_version", "work_id", "evidence_lifecycle"}
|
||||
)
|
||||
_EVIDENCE_KEYS: Final = frozenset(
|
||||
{"runtime_relative_root", "result_id_prefix", "document_name", "schema_version"}
|
||||
)
|
||||
_LIFECYCLE_EVIDENCE_KEYS: Final = frozenset(
|
||||
{"phase", "runtime_relative_root", "result_id_prefix", "document_name", "schema_version"}
|
||||
)
|
||||
|
||||
|
||||
class LaboratoryRegistryError(ValueError):
|
||||
@@ -21,15 +30,15 @@ class LaboratoryRegistryError(ValueError):
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LaboratoryEvidenceDefinition:
|
||||
work_id: str
|
||||
class LaboratoryEvidenceVariant:
|
||||
phase: str
|
||||
runtime_relative_root: PurePosixPath
|
||||
result_id_prefix: str
|
||||
document_name: str
|
||||
result_schema_version: str
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_identifier(self.work_id, "work_id")
|
||||
_identifier(self.phase, "evidence phase")
|
||||
_identifier(self.result_id_prefix, "result_id_prefix")
|
||||
_document_name(self.document_name)
|
||||
_schema_version(self.result_schema_version)
|
||||
@@ -45,6 +54,69 @@ class LaboratoryEvidenceDefinition:
|
||||
return runtime_root.joinpath(*self.runtime_relative_root.parts)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LaboratoryEvidenceDefinition:
|
||||
work_id: str
|
||||
runtime_relative_root: PurePosixPath
|
||||
result_id_prefix: str
|
||||
document_name: str
|
||||
result_schema_version: str
|
||||
lifecycle_variants: tuple[LaboratoryEvidenceVariant, ...] = ()
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_identifier(self.work_id, "work_id")
|
||||
primary = LaboratoryEvidenceVariant(
|
||||
phase="result",
|
||||
runtime_relative_root=self.runtime_relative_root,
|
||||
result_id_prefix=self.result_id_prefix,
|
||||
document_name=self.document_name,
|
||||
result_schema_version=self.result_schema_version,
|
||||
)
|
||||
if not self.lifecycle_variants:
|
||||
return
|
||||
if not all(
|
||||
isinstance(variant, LaboratoryEvidenceVariant)
|
||||
for variant in self.lifecycle_variants
|
||||
):
|
||||
raise LaboratoryRegistryError("LAB lifecycle variants must be immutable evidence")
|
||||
if self.lifecycle_variants[-1] != primary:
|
||||
raise LaboratoryRegistryError("LAB lifecycle terminal evidence must be primary")
|
||||
phases = [variant.phase for variant in self.lifecycle_variants]
|
||||
if len(phases) != len(set(phases)):
|
||||
raise LaboratoryRegistryError("duplicate LAB evidence phase")
|
||||
|
||||
@property
|
||||
def evidence_variants(self) -> tuple[LaboratoryEvidenceVariant, ...]:
|
||||
if self.lifecycle_variants:
|
||||
return self.lifecycle_variants
|
||||
return (
|
||||
LaboratoryEvidenceVariant(
|
||||
phase="result",
|
||||
runtime_relative_root=self.runtime_relative_root,
|
||||
result_id_prefix=self.result_id_prefix,
|
||||
document_name=self.document_name,
|
||||
result_schema_version=self.result_schema_version,
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def result_id_pattern(self) -> re.Pattern[str]:
|
||||
return re.compile(rf"^{re.escape(self.result_id_prefix)}-[a-f0-9]{{64}}$")
|
||||
|
||||
def result_root(self, runtime_root: Path) -> Path:
|
||||
return runtime_root.joinpath(*self.runtime_relative_root.parts)
|
||||
|
||||
def variant_for_result_id(self, result_id: str) -> LaboratoryEvidenceVariant | None:
|
||||
return next(
|
||||
(
|
||||
variant
|
||||
for variant in self.evidence_variants
|
||||
if variant.result_id_pattern.fullmatch(result_id) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LaboratoryEvidenceRegistry:
|
||||
definitions: tuple[LaboratoryEvidenceDefinition, ...]
|
||||
@@ -89,14 +161,42 @@ def _read_definition(path: Path) -> LaboratoryEvidenceDefinition:
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
raise LaboratoryRegistryError(f"LAB definition is unreadable: {path.name}") from exc
|
||||
document = _object(payload, f"LAB definition {path.name}")
|
||||
_exact_keys(document, _TOP_LEVEL_KEYS, f"LAB definition {path.name}")
|
||||
if document["schema_version"] != LABORATORY_EVIDENCE_DEFINITION_SCHEMA:
|
||||
schema_version = document.get("schema_version")
|
||||
if schema_version not in {
|
||||
LABORATORY_EVIDENCE_DEFINITION_SCHEMA,
|
||||
LABORATORY_EVIDENCE_LIFECYCLE_DEFINITION_SCHEMA,
|
||||
}:
|
||||
raise LaboratoryRegistryError(f"LAB definition schema is invalid: {path.name}")
|
||||
expected_keys = (
|
||||
_TOP_LEVEL_KEYS
|
||||
if schema_version == LABORATORY_EVIDENCE_DEFINITION_SCHEMA
|
||||
else _LIFECYCLE_TOP_LEVEL_KEYS
|
||||
)
|
||||
_exact_keys(document, expected_keys, f"LAB definition {path.name}")
|
||||
work_id = _identifier(document["work_id"], "work_id")
|
||||
if path.name != f"{work_id}.json":
|
||||
raise LaboratoryRegistryError(f"LAB definition filename must match work_id: {path.name}")
|
||||
evidence = _object(document["evidence"], f"LAB evidence {work_id}")
|
||||
_exact_keys(evidence, _EVIDENCE_KEYS, f"LAB evidence {work_id}")
|
||||
if schema_version == LABORATORY_EVIDENCE_DEFINITION_SCHEMA:
|
||||
lifecycle_variants: tuple[LaboratoryEvidenceVariant, ...] = ()
|
||||
evidence = _object(document["evidence"], f"LAB evidence {work_id}")
|
||||
_exact_keys(evidence, _EVIDENCE_KEYS, f"LAB evidence {work_id}")
|
||||
else:
|
||||
lifecycle = document["evidence_lifecycle"]
|
||||
if not isinstance(lifecycle, list) or len(lifecycle) < 2:
|
||||
raise LaboratoryRegistryError(
|
||||
f"LAB evidence lifecycle must contain at least two phases: {work_id}"
|
||||
)
|
||||
lifecycle_variants = tuple(
|
||||
_read_variant(row, f"LAB evidence {work_id}[{index}]")
|
||||
for index, row in enumerate(lifecycle)
|
||||
)
|
||||
terminal = lifecycle_variants[-1]
|
||||
evidence = {
|
||||
"runtime_relative_root": str(terminal.runtime_relative_root),
|
||||
"result_id_prefix": terminal.result_id_prefix,
|
||||
"document_name": terminal.document_name,
|
||||
"schema_version": terminal.result_schema_version,
|
||||
}
|
||||
result_id_prefix = _identifier(evidence["result_id_prefix"], "result_id_prefix")
|
||||
document_name = _document_name(evidence["document_name"])
|
||||
result_schema_version = _schema_version(evidence["schema_version"])
|
||||
@@ -106,6 +206,19 @@ def _read_definition(path: Path) -> LaboratoryEvidenceDefinition:
|
||||
result_id_prefix=result_id_prefix,
|
||||
document_name=document_name,
|
||||
result_schema_version=result_schema_version,
|
||||
lifecycle_variants=lifecycle_variants,
|
||||
)
|
||||
|
||||
|
||||
def _read_variant(value: object, label: str) -> LaboratoryEvidenceVariant:
|
||||
evidence = _object(value, label)
|
||||
_exact_keys(evidence, _LIFECYCLE_EVIDENCE_KEYS, label)
|
||||
return LaboratoryEvidenceVariant(
|
||||
phase=_identifier(evidence["phase"], f"{label}.phase"),
|
||||
runtime_relative_root=_relative_root(evidence["runtime_relative_root"]),
|
||||
result_id_prefix=_identifier(evidence["result_id_prefix"], "result_id_prefix"),
|
||||
document_name=_document_name(evidence["document_name"]),
|
||||
result_schema_version=_schema_version(evidence["schema_version"]),
|
||||
)
|
||||
|
||||
|
||||
@@ -177,9 +290,15 @@ def _relative_root(value: object) -> PurePosixPath:
|
||||
def _reject_duplicates(definitions: tuple[LaboratoryEvidenceDefinition, ...]) -> None:
|
||||
dimensions = {
|
||||
"work_id": [definition.work_id for definition in definitions],
|
||||
"result_id_prefix": [definition.result_id_prefix for definition in definitions],
|
||||
"result_id_prefix": [
|
||||
variant.result_id_prefix
|
||||
for definition in definitions
|
||||
for variant in definition.evidence_variants
|
||||
],
|
||||
"runtime_relative_root": [
|
||||
str(definition.runtime_relative_root) for definition in definitions
|
||||
str(variant.runtime_relative_root)
|
||||
for definition in definitions
|
||||
for variant in definition.evidence_variants
|
||||
],
|
||||
}
|
||||
for label, values in dimensions.items():
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import Any, Final
|
||||
from k1link.laboratory.evidence_registry import (
|
||||
LaboratoryEvidenceDefinition,
|
||||
LaboratoryEvidenceRegistry,
|
||||
LaboratoryEvidenceVariant,
|
||||
)
|
||||
|
||||
LABORATORY_EVIDENCE_REPORT_SCHEMA: Final = "missioncore.laboratory-evidence-report/v1"
|
||||
@@ -40,12 +41,13 @@ def verify_laboratory_evidence_result(
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise LaboratoryEvidenceReportError("LAB evidence result is unavailable") from exc
|
||||
if not resolved.is_dir() or definition.result_id_pattern.fullmatch(resolved.name) is None:
|
||||
variant = definition.variant_for_result_id(resolved.name)
|
||||
if not resolved.is_dir() or variant is None:
|
||||
raise LaboratoryEvidenceReportError("LAB evidence result path is invalid")
|
||||
document_path = _safe_file(resolved, definition.document_name)
|
||||
document_path = _safe_file(resolved, variant.document_name)
|
||||
document_bytes = _read_bounded(document_path, _DOCUMENT_MAX_BYTES, "LAB document")
|
||||
document = _json_object(document_bytes, "LAB document")
|
||||
_validate_document(document, definition, resolved.name)
|
||||
_validate_document(document, variant, resolved.name)
|
||||
identity = _object_or_none(document.get("identity"))
|
||||
identity_sha256 = document.get("identity_sha256")
|
||||
if identity is None or not isinstance(identity_sha256, str):
|
||||
@@ -77,13 +79,14 @@ class LaboratoryEvidenceReportService:
|
||||
|
||||
def read(self, work_id: str, result_id: str) -> dict[str, object]:
|
||||
definition = self._definitions.get(work_id)
|
||||
if definition is None or definition.result_id_pattern.fullmatch(result_id) is None:
|
||||
variant = definition.variant_for_result_id(result_id) if definition is not None else None
|
||||
if definition is None or variant is None:
|
||||
raise LaboratoryEvidenceReportNotFound("LAB evidence identity is unknown")
|
||||
result_root = self._result_root(definition, result_id)
|
||||
document_path = _safe_file(result_root, definition.document_name)
|
||||
result_root = self._result_root(variant, result_id)
|
||||
document_path = _safe_file(result_root, variant.document_name)
|
||||
document_bytes = _read_bounded(document_path, _DOCUMENT_MAX_BYTES, "LAB document")
|
||||
document = _json_object(document_bytes, "LAB document")
|
||||
_validate_document(document, definition, result_id)
|
||||
_validate_document(document, variant, result_id)
|
||||
|
||||
identity = _object_or_none(document.get("identity"))
|
||||
identity_sha256 = document.get("identity_sha256")
|
||||
@@ -210,7 +213,7 @@ class LaboratoryEvidenceReportService:
|
||||
|
||||
def _result_root(
|
||||
self,
|
||||
definition: LaboratoryEvidenceDefinition,
|
||||
variant: LaboratoryEvidenceVariant,
|
||||
result_id: str,
|
||||
) -> Path:
|
||||
configured = self._runtime_root_provider()
|
||||
@@ -223,7 +226,7 @@ class LaboratoryEvidenceReportService:
|
||||
runtime_root = runtime_root.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise LaboratoryEvidenceReportNotFound("LAB runtime root is unavailable") from exc
|
||||
candidate = definition.result_root(runtime_root) / result_id
|
||||
candidate = variant.result_root(runtime_root) / result_id
|
||||
if candidate.is_symlink():
|
||||
raise LaboratoryEvidenceReportError("LAB result must not be a symlink")
|
||||
try:
|
||||
@@ -237,7 +240,7 @@ class LaboratoryEvidenceReportService:
|
||||
|
||||
def _validate_document(
|
||||
document: dict[str, Any],
|
||||
definition: LaboratoryEvidenceDefinition,
|
||||
definition: LaboratoryEvidenceVariant,
|
||||
result_id: str,
|
||||
) -> None:
|
||||
if document.get("schema_version") != definition.result_schema_version:
|
||||
|
||||
@@ -169,10 +169,11 @@ class LaboratoryExecutionRegistry:
|
||||
f"laboratory classification is incomplete; missing={missing}, unknown={unknown}"
|
||||
)
|
||||
for definition in self.definitions:
|
||||
if (
|
||||
evidence_by_work_id[definition.work_id].result_schema_version
|
||||
!= definition.evidence_contract
|
||||
):
|
||||
evidence_contracts = {
|
||||
variant.result_schema_version
|
||||
for variant in evidence_by_work_id[definition.work_id].evidence_variants
|
||||
}
|
||||
if definition.evidence_contract not in evidence_contracts:
|
||||
raise LaboratoryExecutionError(
|
||||
f"laboratory evidence contract mismatch: {definition.work_id}"
|
||||
)
|
||||
@@ -311,6 +312,10 @@ class LaboratoryRunner:
|
||||
|
||||
def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]:
|
||||
return {
|
||||
"canonical.m48-small-static-passage-regression/v1": (
|
||||
_run_m48_small_static_passage_regression
|
||||
),
|
||||
"canonical.m48-object-centric-quality/v1": _run_m48_object_centric_quality,
|
||||
"canonical.m4-replay-threat/v1": _run_m4_replay_threat,
|
||||
"canonical.e33-worker-shadow/v1": _run_e33,
|
||||
"canonical.e35-degradation-recovery/v1": _run_e35,
|
||||
@@ -319,6 +324,41 @@ def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]:
|
||||
}
|
||||
|
||||
|
||||
def _run_m48_small_static_passage_regression(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
from k1link.laboratory.m48_small_static_regression import (
|
||||
build_m48_small_static_passage_regression,
|
||||
)
|
||||
|
||||
result = build_m48_small_static_passage_regression(
|
||||
pack_root=request.inputs["pack_root"],
|
||||
correction_session_path=request.inputs["correction_session_path"],
|
||||
profile_path=request.inputs["profile_path"],
|
||||
output_root=request.output_root,
|
||||
)
|
||||
return LaboratoryAdapterResult(
|
||||
result_root=result.result_root,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _run_m48_object_centric_quality(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
from k1link.laboratory.m48_object_quality import score_m48_object_quality
|
||||
|
||||
result = score_m48_object_quality(
|
||||
pack_root=request.inputs["pack_root"],
|
||||
truth_seal_root=request.inputs["truth_seal_root"],
|
||||
output_root=request.output_root,
|
||||
)
|
||||
return LaboratoryAdapterResult(
|
||||
result_root=result.result_root,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _run_m4_replay_threat(request: LaboratoryRunRequest) -> LaboratoryAdapterResult:
|
||||
from k1link.perception.threat_replay import build_threat_replay
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,567 @@
|
||||
"""Deterministic RAVNOVES00 adapter for the M4.8 object-quality pack."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.laboratory.m47_reference_graph import read_m47_reference_graph_lab
|
||||
from k1link.laboratory.m48_object_quality import (
|
||||
M48_PREPARATION_PROVENANCE_SCHEMA,
|
||||
M48_SELECTION_HYPOTHESIS_PROFILE,
|
||||
M48ObjectQualityPack,
|
||||
build_m48_object_quality_pack,
|
||||
)
|
||||
|
||||
M48_SELECTION_SCHEMA: Final = "missioncore.m48-object-quality-selection/v1"
|
||||
M48_SELECTION_ID: Final = "m48-ravnoves00-balanced-connected-clips/v1"
|
||||
M48_SOURCE_ID: Final = "RAVNOVES00"
|
||||
M48_SOURCE_SESSION_ID: Final = "20260720T065719Z_viewer_live"
|
||||
M48_FRAME_COUNT: Final = 4_489
|
||||
M48_IMAGE_WIDTH: Final = 800
|
||||
M48_IMAGE_HEIGHT: Final = 600
|
||||
_CAMERA_INDEX_SCHEMA: Final = "missioncore.camera-recording-index/v1"
|
||||
_GRAPH_FRAME_SCHEMA: Final = "missioncore.local-obstacle-map/v1"
|
||||
_THREAT_FRAME_SCHEMAS: Final = frozenset(
|
||||
{
|
||||
"missioncore.perception-threat-replay-frame/v1",
|
||||
"missioncore.perception-threat-replay-frame/v2",
|
||||
}
|
||||
)
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class M48Ravnoves00PackError(RuntimeError):
|
||||
"""The source adapter escaped the accepted immutable RAVNOVES00 evidence."""
|
||||
|
||||
|
||||
def prepare_m48_ravnoves00_pack(
|
||||
*,
|
||||
m47_lab_root: Path,
|
||||
graph_result_root: Path,
|
||||
threat_result_root: Path,
|
||||
geometry_result_root: Path,
|
||||
camera_index_path: Path,
|
||||
selection_path: Path,
|
||||
frozen_at_utc: str,
|
||||
output_root: Path,
|
||||
) -> M48ObjectQualityPack:
|
||||
"""Freeze the selected M4.8 clips from the exact accepted M4.7 source."""
|
||||
|
||||
lab = read_m47_reference_graph_lab(m47_lab_root)
|
||||
source = _mapping(lab.report.get("source"), "M4.7 source")
|
||||
graph_root = _directory(graph_result_root, "M4.7 graph result")
|
||||
threat_root = _directory(threat_result_root, "M4.6 visual result")
|
||||
geometry_root = _directory(geometry_result_root, "M4.4 geometry result")
|
||||
camera_index = _file(camera_index_path, "recorded camera index")
|
||||
selection = _read_json(_file(selection_path, "M4.8 selection"), "M4.8 selection")
|
||||
clips = _selection_clips(selection)
|
||||
|
||||
if (
|
||||
graph_root.name != source.get("graph_result_id")
|
||||
or threat_root.name != source.get("visual_result_id")
|
||||
or source.get("source_id") != M48_SOURCE_ID
|
||||
or source.get("source_session_id") != M48_SOURCE_SESSION_ID
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.8 source roots do not match the accepted M4.7 LAB")
|
||||
|
||||
graph_frames_path = _validate_graph_result(graph_root)
|
||||
threat_frames_path, threat_identity = _validate_threat_result(
|
||||
threat_root,
|
||||
expected_frames_sha256=source.get("threat_frames_sha256"),
|
||||
)
|
||||
geometry_frames_path = _validate_geometry_result(
|
||||
geometry_root,
|
||||
expected_result_id=threat_identity.get("geometry_result_id"),
|
||||
expected_frames_sha256=threat_identity.get("geometry_frames_sha256"),
|
||||
)
|
||||
camera_rows = tuple(_iter_jsonl(camera_index, "recorded camera index"))
|
||||
_validate_camera_rows(camera_rows)
|
||||
|
||||
selected_sequences = {
|
||||
sequence
|
||||
for clip in clips
|
||||
for sequence in range(
|
||||
_integer(clip.get("start_sequence"), "clip start_sequence"),
|
||||
_integer(clip.get("end_sequence"), "clip end_sequence") + 1,
|
||||
)
|
||||
}
|
||||
frame_catalog: list[dict[str, object]] = []
|
||||
predictions: list[dict[str, object]] = []
|
||||
previous_source_time_ns = -1
|
||||
graph_rows = _iter_jsonl(graph_frames_path, "M4.7 graph frames")
|
||||
threat_rows = _iter_jsonl(threat_frames_path, "M4.6 threat frames")
|
||||
geometry_rows = _iter_jsonl(geometry_frames_path, "M4.4 geometry frames")
|
||||
for frame_index, values in enumerate(
|
||||
zip_longest(graph_rows, threat_rows, geometry_rows, camera_rows),
|
||||
):
|
||||
graph_row, threat_row, geometry_row, camera_row = values
|
||||
if graph_row is None or threat_row is None or geometry_row is None or camera_row is None:
|
||||
raise M48Ravnoves00PackError("M4.8 source ledgers have different lengths")
|
||||
sequence = frame_index + 1
|
||||
source_time_ns = _validate_bound_frame(
|
||||
graph_row=graph_row,
|
||||
threat_row=threat_row,
|
||||
geometry_row=geometry_row,
|
||||
camera_row=camera_row,
|
||||
frame_index=frame_index,
|
||||
previous_source_time_ns=previous_source_time_ns,
|
||||
)
|
||||
previous_source_time_ns = source_time_ns
|
||||
frame_catalog.append(
|
||||
{
|
||||
"sequence": sequence,
|
||||
"source_time_ns": source_time_ns,
|
||||
"camera_fragment_sha256": camera_row["sha256"],
|
||||
}
|
||||
)
|
||||
if sequence in selected_sequences:
|
||||
obstacle_map = _mapping(graph_row.get("obstacle_map"), "M4.7 obstacle map")
|
||||
predictions.append(
|
||||
{
|
||||
"sequence": sequence,
|
||||
"source_time_ns": source_time_ns,
|
||||
"terminal_outcome": "delivered",
|
||||
"terminal_reason": None,
|
||||
"free_space_claimed": obstacle_map["free_space_claimed"],
|
||||
"objects": _prediction_objects(
|
||||
threat_row.get("camera_proposals"),
|
||||
geometry_observations=geometry_row.get("observations"),
|
||||
metric_obstacles=threat_row.get("metric_obstacles"),
|
||||
),
|
||||
}
|
||||
)
|
||||
if len(frame_catalog) != M48_FRAME_COUNT:
|
||||
raise M48Ravnoves00PackError("M4.8 source frame count changed")
|
||||
|
||||
preparation_provenance = {
|
||||
"schema_version": M48_PREPARATION_PROVENANCE_SCHEMA,
|
||||
"adapter": {
|
||||
"module": "k1link.laboratory.m48_ravnoves00_pack",
|
||||
"sha256": _file_sha256(Path(__file__).resolve(strict=True)),
|
||||
},
|
||||
"selection": {
|
||||
"selection_id": M48_SELECTION_ID,
|
||||
"sha256": _file_sha256(selection_path),
|
||||
},
|
||||
"camera_index": {
|
||||
"source_session_id": M48_SOURCE_SESSION_ID,
|
||||
"sha256": _file_sha256(camera_index),
|
||||
"byte_length": camera_index.stat().st_size,
|
||||
"frame_count": len(camera_rows),
|
||||
},
|
||||
"graph": _source_provenance(graph_root, graph_frames_path),
|
||||
"threat": _source_provenance(threat_root, threat_frames_path),
|
||||
"geometry": _source_provenance(geometry_root, geometry_frames_path),
|
||||
}
|
||||
|
||||
return build_m48_object_quality_pack(
|
||||
m47_lab_root=lab.result_root,
|
||||
frame_catalog=frame_catalog,
|
||||
clips=clips,
|
||||
predictions=predictions,
|
||||
preparation_provenance=preparation_provenance,
|
||||
frozen_at_utc=frozen_at_utc,
|
||||
output_root=output_root,
|
||||
)
|
||||
|
||||
|
||||
def _validate_graph_result(root: Path) -> Path:
|
||||
manifest = _read_json(_file(root / "manifest.json", "M4.7 graph manifest"), "graph manifest")
|
||||
files = _mapping(manifest.get("files"), "M4.7 graph files")
|
||||
descriptor = _mapping(files.get("frames.jsonl"), "M4.7 graph frame descriptor")
|
||||
frames = _file(root / "frames.jsonl", "M4.7 graph frames")
|
||||
expected_bytes = descriptor.get("bytes")
|
||||
expected_sha256 = descriptor.get("sha256")
|
||||
if (
|
||||
manifest.get("schema_version") != "missioncore.reference-perception-graph-manifest/v1"
|
||||
or manifest.get("result_id") != root.name
|
||||
or manifest.get("accepted") is not True
|
||||
or manifest.get("graph_id") != "reference-perception-graph/v2"
|
||||
or manifest.get("run_mode") != "lossless-replay"
|
||||
or not isinstance(expected_bytes, int)
|
||||
or expected_bytes != frames.stat().st_size
|
||||
or not _is_sha256(expected_sha256)
|
||||
or _file_sha256(frames) != expected_sha256
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.7 graph result changed")
|
||||
return frames
|
||||
|
||||
|
||||
def _validate_threat_result(
|
||||
root: Path,
|
||||
*,
|
||||
expected_frames_sha256: object,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
manifest = _read_json(
|
||||
_file(root / "manifest.json", "M4.6 threat manifest"),
|
||||
"threat manifest",
|
||||
)
|
||||
identity = _mapping(manifest.get("identity"), "M4.6 threat identity")
|
||||
frames = _file(root / "frames.jsonl", "M4.6 threat frames")
|
||||
if (
|
||||
manifest.get("schema_version") != "missioncore.perception-threat-replay-result/v2"
|
||||
or manifest.get("result_id") != root.name
|
||||
or manifest.get("accepted") is not True
|
||||
or identity.get("source_session_id") != M48_SOURCE_SESSION_ID
|
||||
or not _is_sha256(expected_frames_sha256)
|
||||
or identity.get("frames_sha256") != expected_frames_sha256
|
||||
or _file_sha256(frames) != expected_frames_sha256
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.6 threat result changed")
|
||||
return frames, identity
|
||||
|
||||
|
||||
def _validate_geometry_result(
|
||||
root: Path,
|
||||
*,
|
||||
expected_result_id: object,
|
||||
expected_frames_sha256: object,
|
||||
) -> Path:
|
||||
manifest = _read_json(
|
||||
_file(root / "manifest.json", "M4.4 geometry manifest"),
|
||||
"geometry manifest",
|
||||
)
|
||||
identity = _mapping(manifest.get("identity"), "M4.4 geometry identity")
|
||||
frames = _file(root / "frames.jsonl", "M4.4 geometry frames")
|
||||
if (
|
||||
manifest.get("schema_version") != "missioncore.perception-geometry-replay-result/v1"
|
||||
or root.name != expected_result_id
|
||||
or identity.get("accepted") is not True
|
||||
or identity.get("source_pack_id")
|
||||
!= "e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b"
|
||||
or not _is_sha256(expected_frames_sha256)
|
||||
or identity.get("frames_sha256") != expected_frames_sha256
|
||||
or _file_sha256(frames) != expected_frames_sha256
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.4 geometry result changed")
|
||||
return frames
|
||||
|
||||
|
||||
def _selection_clips(document: Mapping[str, object]) -> tuple[dict[str, object], ...]:
|
||||
expected_keys = {
|
||||
"schema_version",
|
||||
"selection_id",
|
||||
"source_id",
|
||||
"source_session_id",
|
||||
"selection_basis",
|
||||
"camera_frame_size",
|
||||
"selection_hypothesis_profile",
|
||||
"clips",
|
||||
}
|
||||
frame_size = _mapping(document.get("camera_frame_size"), "selection frame size")
|
||||
raw_clips = document.get("clips")
|
||||
if (
|
||||
set(document) != expected_keys
|
||||
or document.get("schema_version") != M48_SELECTION_SCHEMA
|
||||
or document.get("selection_id") != M48_SELECTION_ID
|
||||
or document.get("source_id") != M48_SOURCE_ID
|
||||
or document.get("source_session_id") != M48_SOURCE_SESSION_ID
|
||||
or document.get("selection_basis")
|
||||
!= "prediction-frozen-source-curation-before-independent-truth"
|
||||
or frame_size != {"width": M48_IMAGE_WIDTH, "height": M48_IMAGE_HEIGHT}
|
||||
or document.get("selection_hypothesis_profile") != M48_SELECTION_HYPOTHESIS_PROFILE
|
||||
or not isinstance(raw_clips, list)
|
||||
or any(not isinstance(item, dict) for item in raw_clips)
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.8 selection contract changed")
|
||||
return tuple(dict(item) for item in raw_clips)
|
||||
|
||||
|
||||
def _source_provenance(root: Path, frames_path: Path) -> dict[str, str]:
|
||||
return {
|
||||
"result_id": root.name,
|
||||
"manifest_sha256": _file_sha256(root / "manifest.json"),
|
||||
"frames_sha256": _file_sha256(frames_path),
|
||||
}
|
||||
|
||||
|
||||
def _validate_camera_rows(rows: tuple[dict[str, Any], ...]) -> None:
|
||||
if len(rows) != M48_FRAME_COUNT:
|
||||
raise M48Ravnoves00PackError("recorded camera index frame count changed")
|
||||
previous_session_time = -1
|
||||
for expected_sequence, row in enumerate(rows, start=1):
|
||||
session_time = row.get("session_monotonic_ns")
|
||||
if (
|
||||
row.get("schema_version") != _CAMERA_INDEX_SCHEMA
|
||||
or row.get("kind") != "media"
|
||||
or row.get("sequence") != expected_sequence
|
||||
or not isinstance(session_time, int)
|
||||
or session_time <= previous_session_time
|
||||
or not _is_sha256(row.get("sha256"))
|
||||
):
|
||||
raise M48Ravnoves00PackError("recorded camera index changed")
|
||||
previous_session_time = session_time
|
||||
|
||||
|
||||
def _validate_bound_frame(
|
||||
*,
|
||||
graph_row: Mapping[str, Any],
|
||||
threat_row: Mapping[str, Any],
|
||||
geometry_row: Mapping[str, Any],
|
||||
camera_row: Mapping[str, Any],
|
||||
frame_index: int,
|
||||
previous_source_time_ns: int,
|
||||
) -> int:
|
||||
obstacle_map = _mapping(graph_row.get("obstacle_map"), "M4.7 obstacle map")
|
||||
source_time_ns = threat_row.get("source_time_ns")
|
||||
if (
|
||||
graph_row.get("sequence") != frame_index
|
||||
or obstacle_map.get("schema_version") != _GRAPH_FRAME_SCHEMA
|
||||
or obstacle_map.get("frame_id") != f"frame-{frame_index:06d}"
|
||||
or not isinstance(obstacle_map.get("free_space_claimed"), bool)
|
||||
or threat_row.get("schema_version") not in _THREAT_FRAME_SCHEMAS
|
||||
or threat_row.get("sequence") != frame_index
|
||||
or threat_row.get("frame_id") != f"frame-{frame_index:06d}"
|
||||
or geometry_row.get("schema_version") != "missioncore.perception-geometry-replay-frame/v1"
|
||||
or geometry_row.get("sequence") != frame_index
|
||||
or geometry_row.get("frame_id") != f"frame-{frame_index:06d}"
|
||||
or geometry_row.get("source_available") != threat_row.get("source_available")
|
||||
or not isinstance(source_time_ns, int)
|
||||
or source_time_ns <= previous_source_time_ns
|
||||
or camera_row.get("sequence") != frame_index + 1
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.8 frame binding changed")
|
||||
return source_time_ns
|
||||
|
||||
|
||||
def _prediction_objects(
|
||||
value: object,
|
||||
*,
|
||||
geometry_observations: object,
|
||||
metric_obstacles: object,
|
||||
) -> list[dict[str, object]]:
|
||||
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
||||
raise M48Ravnoves00PackError("M4.6 camera proposal collection changed")
|
||||
observations = _proposal_observations(geometry_observations)
|
||||
obstacles = _metric_obstacles(metric_obstacles)
|
||||
objects: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
for proposal in value:
|
||||
prediction_id = proposal.get("proposal_id")
|
||||
occupied_support = proposal.get("occupied_support")
|
||||
threat_value = proposal.get("threat_decision")
|
||||
if (
|
||||
not isinstance(prediction_id, str)
|
||||
or prediction_id in seen
|
||||
or not isinstance(occupied_support, bool)
|
||||
or threat_value not in {None, "threat", "not-threat", "unknown"}
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.6 camera proposal identity changed")
|
||||
seen.add(prediction_id)
|
||||
observation = observations.get(prediction_id)
|
||||
geometry = "associated" if occupied_support else "unknown"
|
||||
freshness = "current"
|
||||
motion = "unsupported"
|
||||
threat = threat_value if isinstance(threat_value, str) else "unknown"
|
||||
if occupied_support:
|
||||
if observation is None:
|
||||
raise M48Ravnoves00PackError("associated proposal lost its geometry observation")
|
||||
currentness = observation.get("currentness")
|
||||
if currentness not in {"current", "held", "stale", "unavailable"}:
|
||||
raise M48Ravnoves00PackError("associated proposal currentness changed")
|
||||
freshness = str(currentness)
|
||||
centroid = _metric_centroid(observation)
|
||||
obstacle = _match_metric_obstacle(centroid, obstacles)
|
||||
raw_motion = obstacle.get("motion")
|
||||
motion = {
|
||||
"moving": "moving",
|
||||
"stationary": "static",
|
||||
"unknown": "unknown",
|
||||
}.get(str(raw_motion), "")
|
||||
assessment = _mapping(obstacle.get("assessment"), "metric obstacle assessment")
|
||||
obstacle_threat = assessment.get("decision")
|
||||
if not motion or obstacle_threat not in {"threat", "not-threat", "unknown"}:
|
||||
raise M48Ravnoves00PackError("associated proposal state changed")
|
||||
threat = str(obstacle_threat)
|
||||
causes: set[str] = set()
|
||||
if geometry == "unknown":
|
||||
causes.add("insufficient-geometry-support")
|
||||
if threat == "unknown":
|
||||
causes.add("threat-evidence-insufficient")
|
||||
if motion == "unknown":
|
||||
causes.add("motion-not-supported")
|
||||
objects.append(
|
||||
{
|
||||
"prediction_id": prediction_id,
|
||||
"extent_xyxy": _normalized_extent(proposal.get("bbox_xyxy")),
|
||||
"geometry_association": geometry,
|
||||
"freshness": freshness,
|
||||
"motion": motion,
|
||||
"threat": threat,
|
||||
"unknown_causes": sorted(causes),
|
||||
}
|
||||
)
|
||||
return objects
|
||||
|
||||
|
||||
def _proposal_observations(value: object) -> dict[str, dict[str, Any]]:
|
||||
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
||||
raise M48Ravnoves00PackError("M4.4 observation collection changed")
|
||||
mapped: dict[str, dict[str, Any]] = {}
|
||||
for observation in value:
|
||||
proposal_ids = observation.get("proposal_ids")
|
||||
if not isinstance(proposal_ids, list) or any(
|
||||
not isinstance(item, str) for item in proposal_ids
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.4 proposal binding changed")
|
||||
for proposal_id in proposal_ids:
|
||||
if proposal_id in mapped:
|
||||
raise M48Ravnoves00PackError("M4.4 proposal has multiple observations")
|
||||
mapped[proposal_id] = observation
|
||||
return mapped
|
||||
|
||||
|
||||
def _metric_obstacles(value: object) -> tuple[dict[str, Any], ...]:
|
||||
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
||||
raise M48Ravnoves00PackError("M4.6 metric obstacle collection changed")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _metric_centroid(observation: Mapping[str, Any]) -> tuple[float, float, float]:
|
||||
geometry = _mapping(observation.get("metric_geometry"), "proposal metric geometry")
|
||||
value = geometry.get("centroid_xyz_m")
|
||||
if (
|
||||
not isinstance(value, list)
|
||||
or len(value) != 3
|
||||
or any(
|
||||
not isinstance(item, (int, float))
|
||||
or isinstance(item, bool)
|
||||
or not math.isfinite(float(item))
|
||||
for item in value
|
||||
)
|
||||
):
|
||||
raise M48Ravnoves00PackError("proposal metric centroid changed")
|
||||
return float(value[0]), float(value[1]), float(value[2])
|
||||
|
||||
|
||||
def _match_metric_obstacle(
|
||||
centroid: tuple[float, float, float],
|
||||
obstacles: tuple[dict[str, Any], ...],
|
||||
) -> dict[str, Any]:
|
||||
matches: list[dict[str, Any]] = []
|
||||
for obstacle in obstacles:
|
||||
value = obstacle.get("centroid_map_xyz_m")
|
||||
if (
|
||||
isinstance(value, list)
|
||||
and len(value) == 3
|
||||
and all(isinstance(item, (int, float)) and not isinstance(item, bool) for item in value)
|
||||
and max(
|
||||
abs(float(left) - float(right)) for left, right in zip(value, centroid, strict=True)
|
||||
)
|
||||
<= 1e-9
|
||||
):
|
||||
matches.append(obstacle)
|
||||
if len(matches) != 1:
|
||||
raise M48Ravnoves00PackError("proposal metric obstacle association is ambiguous")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _normalized_extent(value: object) -> list[float]:
|
||||
if (
|
||||
not isinstance(value, list)
|
||||
or len(value) != 4
|
||||
or any(
|
||||
not isinstance(item, (int, float))
|
||||
or isinstance(item, bool)
|
||||
or not math.isfinite(float(item))
|
||||
for item in value
|
||||
)
|
||||
):
|
||||
raise M48Ravnoves00PackError("M4.6 proposal extent changed")
|
||||
x_min, y_min, x_max, y_max = (float(item) for item in value)
|
||||
extent = [
|
||||
x_min / M48_IMAGE_WIDTH,
|
||||
y_min / M48_IMAGE_HEIGHT,
|
||||
x_max / M48_IMAGE_WIDTH,
|
||||
y_max / M48_IMAGE_HEIGHT,
|
||||
]
|
||||
if not 0.0 <= extent[0] < extent[2] <= 1.0 or not 0.0 <= extent[1] < extent[3] <= 1.0:
|
||||
raise M48Ravnoves00PackError("M4.6 proposal extent escaped the camera raster")
|
||||
return extent
|
||||
|
||||
|
||||
def _iter_jsonl(path: Path, label: str) -> Iterator[dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
for line_number, line in enumerate(stream, start=1):
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise M48Ravnoves00PackError(f"{label} row {line_number} is invalid") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48Ravnoves00PackError(f"{label} row {line_number} is not an object")
|
||||
yield value
|
||||
|
||||
|
||||
def _read_json(path: Path, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise M48Ravnoves00PackError(f"{label} is invalid") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48Ravnoves00PackError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _mapping(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise M48Ravnoves00PackError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _integer(value: object, label: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool):
|
||||
raise M48Ravnoves00PackError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
raise M48Ravnoves00PackError(f"{label} must not be a symlink")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M48Ravnoves00PackError(f"{label} is unavailable") from exc
|
||||
if not resolved.is_dir():
|
||||
raise M48Ravnoves00PackError(f"{label} is unavailable")
|
||||
return resolved
|
||||
|
||||
|
||||
def _file(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
raise M48Ravnoves00PackError(f"{label} must not be a symlink")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M48Ravnoves00PackError(f"{label} is unavailable") from exc
|
||||
if not resolved.is_file():
|
||||
raise M48Ravnoves00PackError(f"{label} is unavailable")
|
||||
return resolved
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _is_sha256(value: object) -> bool:
|
||||
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"M48Ravnoves00PackError",
|
||||
"M48_SELECTION_ID",
|
||||
"M48_SELECTION_SCHEMA",
|
||||
"prepare_m48_ravnoves00_pack",
|
||||
]
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Prediction-free raw spatial evidence for the neutral M4.8 review surface.
|
||||
|
||||
This reader deliberately does not open the M4.7 graph payload or the frozen M4.8
|
||||
prediction ledger. It reuses the already verified recorded-geometry and replay
|
||||
body-frame primitives to expose only a bounded current LiDAR increment in the
|
||||
virtual body frame, together with immutable rig/corridor parameters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.laboratory.m47_reference_graph import (
|
||||
M47ReferenceGraphLabError,
|
||||
read_m47_reference_graph_lab,
|
||||
)
|
||||
from k1link.laboratory.m48_object_quality import M48ObjectQualityPack
|
||||
from k1link.perception.spatial_evidence import (
|
||||
SpatialEvidenceProjectionError,
|
||||
sample_points_in_body_frame,
|
||||
)
|
||||
from k1link.perception.threat_replay import (
|
||||
ThreatReplayError,
|
||||
ThreatReplayResult,
|
||||
read_threat_replay_result,
|
||||
)
|
||||
from k1link.perception.threat_timeline import (
|
||||
RECORDED_SPATIAL_POINT_LIMIT,
|
||||
RecordedThreatTimeline,
|
||||
RecordedThreatTimelineError,
|
||||
)
|
||||
|
||||
M48_RAW_SPATIAL_FRAME_SCHEMA: Final = "missioncore.m48-neutral-object-review-spatial-frame/v1"
|
||||
M48_EXPECTED_SOURCE_ID: Final = "RAVNOVES00"
|
||||
M48_EXPECTED_E10_SOURCE_ID: Final = "sensor.camera.right"
|
||||
M48_EXPECTED_SESSION_ID: Final = "20260720T065719Z_viewer_live"
|
||||
M48_EXPECTED_FRAME_COUNT: Final = 4_489
|
||||
M48_EXPECTED_SOURCE_PACK_ID: Final = (
|
||||
"e10-lidar-pack-576c994a6c814e2592dd6240ace3902a5db94843312c759a73ba0c9166157d2b"
|
||||
)
|
||||
M48_EXPECTED_SOURCE_PACK_SHA256: Final = (
|
||||
"0685d24219d8236caf8b7f1685e93f6d6b59e7fd015a768d88a92bbe8b154944"
|
||||
)
|
||||
M48_EXPECTED_THREAT_RESULT_ID: Final = (
|
||||
"m4-threat-replay-2a953c5f27f2a5b1dddc5c658c1de2c323d7796084a099c024987a1da03aa324"
|
||||
)
|
||||
|
||||
_E10_SCHEMA: Final = "missioncore.e10-lidar-replay-pack/v1"
|
||||
_E10_ARTIFACT_NAME: Final = "lidar-pack.npz"
|
||||
_FALSE_AUTHORITY: Final = {
|
||||
"mode": "replay-simulated",
|
||||
"physical_live": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
_MANIFEST_KEYS: Final = {
|
||||
"artifact",
|
||||
"classification",
|
||||
"created_at_utc",
|
||||
"ground_truth",
|
||||
"identity",
|
||||
"identity_sha256",
|
||||
"pack_id",
|
||||
"schema_version",
|
||||
}
|
||||
_IDENTITY_KEYS: Final = {
|
||||
"available_lidar_frames",
|
||||
"calibration_sha256",
|
||||
"camera_slot",
|
||||
"e6_profile_sha256",
|
||||
"e6_result_id",
|
||||
"frame_count",
|
||||
"input_sha256",
|
||||
"job_id",
|
||||
"point_count",
|
||||
"producer_sha256",
|
||||
"projection",
|
||||
"schema_version",
|
||||
"semantic_timeline_result_id",
|
||||
"session_id",
|
||||
"source_end_frame_index",
|
||||
"source_id",
|
||||
"source_start_frame_index",
|
||||
"temporal_binding",
|
||||
"temporal_policy",
|
||||
"timeline_end_seconds",
|
||||
"timeline_start_seconds",
|
||||
}
|
||||
_ARTIFACT_KEYS: Final = {"byte_length", "media_type", "path", "sha256"}
|
||||
|
||||
|
||||
class M48RawEvidenceError(RuntimeError):
|
||||
"""Neutral M4.8 spatial evidence escaped an immutable source binding."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PackFrameBinding:
|
||||
clip_id: str
|
||||
source_time_ns: int
|
||||
|
||||
|
||||
class M48RawEvidenceReader:
|
||||
"""Provide one prediction-blind, bounded body-frame projection per call.
|
||||
|
||||
Construct production instances with :meth:`from_repository`. The object is
|
||||
directly compatible with the M4.8 API provider callable:
|
||||
``reader(pack, one_based_sequence)``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repository_root: Path,
|
||||
threat_result: ThreatReplayResult,
|
||||
timeline: RecordedThreatTimeline,
|
||||
point_limit: int = RECORDED_SPATIAL_POINT_LIMIT,
|
||||
) -> None:
|
||||
if (
|
||||
not isinstance(point_limit, int)
|
||||
or isinstance(point_limit, bool)
|
||||
or not 1 <= point_limit <= RECORDED_SPATIAL_POINT_LIMIT
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 raw evidence point limit is invalid")
|
||||
self.repository_root = repository_root.resolve(strict=True)
|
||||
self.threat_result = threat_result
|
||||
self.timeline = timeline
|
||||
self.point_limit = point_limit
|
||||
self._pack_indices: dict[str, dict[int, _PackFrameBinding]] = {}
|
||||
self._lock = RLock()
|
||||
|
||||
@classmethod
|
||||
def from_repository(
|
||||
cls,
|
||||
*,
|
||||
repository_root: Path,
|
||||
threat_result_root: Path,
|
||||
expected_source_pack_id: str = M48_EXPECTED_SOURCE_PACK_ID,
|
||||
point_limit: int = RECORDED_SPATIAL_POINT_LIMIT,
|
||||
) -> M48RawEvidenceReader:
|
||||
"""Open the exact sealed M4 result and its exact E10 source generation.
|
||||
|
||||
``threat_result_root`` is the immutable result generation directory, not
|
||||
the parent collection. No latest-by-mtime discovery is permitted.
|
||||
"""
|
||||
|
||||
repository = _strict_directory(repository_root, "repository root")
|
||||
if expected_source_pack_id != M48_EXPECTED_SOURCE_PACK_ID:
|
||||
raise M48RawEvidenceError("M4.8 E10 pack id escaped the canonical binding")
|
||||
threat_root = _strict_directory(threat_result_root, "threat result root")
|
||||
try:
|
||||
result = read_threat_replay_result(threat_root)
|
||||
except (OSError, ValueError, ThreatReplayError) as exc:
|
||||
raise M48RawEvidenceError("M4.8 threat result is invalid") from exc
|
||||
_validate_threat_result(result, expected_source_pack_id=expected_source_pack_id)
|
||||
pack_root = (
|
||||
repository / ".runtime/compute-experiments/e10/lidar-packs" / expected_source_pack_id
|
||||
)
|
||||
_validate_e10_pack(
|
||||
pack_root,
|
||||
expected_pack_id=expected_source_pack_id,
|
||||
expected_artifact_sha256=M48_EXPECTED_SOURCE_PACK_SHA256,
|
||||
)
|
||||
try:
|
||||
timeline = RecordedThreatTimeline(repository_root=repository, result=result)
|
||||
except (OSError, ValueError, RecordedThreatTimelineError) as exc:
|
||||
raise M48RawEvidenceError("M4.8 recorded geometry timeline is invalid") from exc
|
||||
if (
|
||||
len(timeline.index.source_times_ns) != M48_EXPECTED_FRAME_COUNT
|
||||
or timeline.profile.source_id != M48_EXPECTED_SOURCE_ID
|
||||
or timeline.profile.session_id != M48_EXPECTED_SESSION_ID
|
||||
or timeline.profile.source_pack_id != expected_source_pack_id
|
||||
or timeline.profile.source_pack_sha256 != M48_EXPECTED_SOURCE_PACK_SHA256
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 recorded geometry binding changed")
|
||||
return cls(
|
||||
repository_root=repository,
|
||||
threat_result=result,
|
||||
timeline=timeline,
|
||||
point_limit=point_limit,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
pack: M48ObjectQualityPack,
|
||||
sequence: int,
|
||||
) -> dict[str, object]:
|
||||
return self.frame(pack=pack, sequence=sequence)
|
||||
|
||||
def frame(
|
||||
self,
|
||||
*,
|
||||
pack: M48ObjectQualityPack,
|
||||
sequence: int,
|
||||
) -> dict[str, object]:
|
||||
"""Return one one-based, clip-bound neutral spatial frame."""
|
||||
|
||||
if (
|
||||
not isinstance(sequence, int)
|
||||
or isinstance(sequence, bool)
|
||||
or not 1 <= sequence <= M48_EXPECTED_FRAME_COUNT
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 raw evidence sequence is invalid")
|
||||
binding = self._binding_for(pack, sequence)
|
||||
frame_index = sequence - 1
|
||||
try:
|
||||
temporal = self.timeline.store.temporal_binding_for_index(frame_index)
|
||||
body_frame = self.timeline.body_frames.body_frame_for_frame(f"frame-{frame_index:06d}")
|
||||
except (RuntimeError, TypeError, ValueError) as exc:
|
||||
raise M48RawEvidenceError("M4.8 source frame binding is invalid") from exc
|
||||
if temporal.frame_index != frame_index or temporal.source_time_ns != binding.source_time_ns:
|
||||
raise M48RawEvidenceError("M4.8 source time escaped the neutral frame reference")
|
||||
|
||||
points_body: list[list[float]] = []
|
||||
if body_frame is not None:
|
||||
if not temporal.source_available:
|
||||
raise M48RawEvidenceError("unavailable source produced an M4.8 body frame")
|
||||
points = self.timeline.store.current_points_for_frame(frame_index)
|
||||
if points is None:
|
||||
raise M48RawEvidenceError("qualified M4.8 body frame lacks current LiDAR")
|
||||
try:
|
||||
points_body, _ = sample_points_in_body_frame(
|
||||
points,
|
||||
body_frame,
|
||||
point_limit=self.point_limit,
|
||||
)
|
||||
except SpatialEvidenceProjectionError as exc:
|
||||
raise M48RawEvidenceError("M4.8 body-frame point projection failed") from exc
|
||||
|
||||
profile = self.timeline.profile
|
||||
return {
|
||||
"schema_version": M48_RAW_SPATIAL_FRAME_SCHEMA,
|
||||
"pack_id": pack.result_id,
|
||||
"clip_id": binding.clip_id,
|
||||
"sequence": sequence,
|
||||
"source_time_ns": temporal.source_time_ns,
|
||||
"source_available": temporal.source_available,
|
||||
"body_frame_available": body_frame is not None,
|
||||
"point_cloud_body_xyz_m": points_body,
|
||||
"rig": {
|
||||
"profile_id": profile.rig.profile_id,
|
||||
"length_m": profile.rig.body_length_m,
|
||||
"width_m": profile.rig.body_width_m,
|
||||
"lidar_reference": profile.rig.lidar_reference,
|
||||
"nominal_sensor_height_m": profile.rig.nominal_sensor_height_m,
|
||||
"physical_mount_claimed": False,
|
||||
},
|
||||
"corridor": {
|
||||
"profile_id": profile.corridor.profile_id,
|
||||
"forward_length_m": profile.corridor.forward_length_m,
|
||||
"rear_margin_m": profile.corridor.rear_margin_m,
|
||||
"lateral_clearance_m": profile.corridor.lateral_clearance_m,
|
||||
"half_width_m": (
|
||||
profile.rig.body_width_m / 2 + profile.corridor.lateral_clearance_m
|
||||
),
|
||||
"prediction_horizon_seconds": (profile.corridor.prediction_horizon_seconds),
|
||||
},
|
||||
"occupied_voxel_size_m": profile.corridor.occupied_voxel_size_m,
|
||||
"candidate_identity_included": False,
|
||||
"graph_boxes_ids_scores_included": False,
|
||||
"frozen_predictions_included": False,
|
||||
"strata_included": False,
|
||||
"authority": dict(_FALSE_AUTHORITY),
|
||||
}
|
||||
|
||||
def _binding_for(
|
||||
self,
|
||||
pack: M48ObjectQualityPack,
|
||||
sequence: int,
|
||||
) -> _PackFrameBinding:
|
||||
with self._lock:
|
||||
index = self._pack_indices.get(pack.result_id)
|
||||
if index is None:
|
||||
_validate_m47_pack_binding(
|
||||
repository_root=self.repository_root,
|
||||
pack=pack,
|
||||
threat_result=self.threat_result,
|
||||
)
|
||||
index = _index_neutral_frame_references(pack)
|
||||
self._pack_indices[pack.result_id] = index
|
||||
binding = index.get(sequence)
|
||||
if binding is None:
|
||||
raise M48RawEvidenceError("M4.8 sequence is outside the selected neutral clips")
|
||||
return binding
|
||||
|
||||
|
||||
def _validate_threat_result(
|
||||
result: ThreatReplayResult,
|
||||
*,
|
||||
expected_source_pack_id: str,
|
||||
) -> None:
|
||||
identity = result.manifest.get("identity")
|
||||
metrics = identity.get("metrics") if isinstance(identity, dict) else None
|
||||
frames = metrics.get("frames") if isinstance(metrics, dict) else None
|
||||
if (
|
||||
result.result_id != M48_EXPECTED_THREAT_RESULT_ID
|
||||
or result.result_root.name != result.result_id
|
||||
or result.accepted is not True
|
||||
or not isinstance(identity, dict)
|
||||
or identity.get("source_id") != M48_EXPECTED_SOURCE_ID
|
||||
or identity.get("source_session_id") != M48_EXPECTED_SESSION_ID
|
||||
or identity.get("source_pack_id") != expected_source_pack_id
|
||||
or identity.get("source_pack_sha256") != M48_EXPECTED_SOURCE_PACK_SHA256
|
||||
or not isinstance(frames, dict)
|
||||
or frames.get("total") != M48_EXPECTED_FRAME_COUNT
|
||||
or identity.get("authority")
|
||||
!= {
|
||||
**_FALSE_AUTHORITY,
|
||||
"physical_collision_accepted": False,
|
||||
"ground_truth": False,
|
||||
}
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 threat result escaped the canonical source")
|
||||
|
||||
|
||||
def _validate_e10_pack(
|
||||
pack_root: Path,
|
||||
*,
|
||||
expected_pack_id: str,
|
||||
expected_artifact_sha256: str,
|
||||
) -> Path:
|
||||
root = _strict_directory(pack_root, "E10 pack root")
|
||||
if root.name != expected_pack_id:
|
||||
raise M48RawEvidenceError("E10 pack path escaped its expected identity")
|
||||
manifest_path = root / "manifest.json"
|
||||
if (
|
||||
manifest_path.is_symlink()
|
||||
or not manifest_path.is_file()
|
||||
or manifest_path.resolve(strict=True).parent != root
|
||||
):
|
||||
raise M48RawEvidenceError("E10 manifest path is invalid")
|
||||
manifest = _read_json(manifest_path, "E10 manifest")
|
||||
if set(manifest) != _MANIFEST_KEYS:
|
||||
raise M48RawEvidenceError("E10 manifest fields changed")
|
||||
identity = _mapping(manifest.get("identity"), "E10 identity")
|
||||
artifact = _mapping(manifest.get("artifact"), "E10 artifact")
|
||||
if set(identity) != _IDENTITY_KEYS or set(artifact) != _ARTIFACT_KEYS:
|
||||
raise M48RawEvidenceError("E10 identity or artifact fields changed")
|
||||
identity_sha256 = _canonical_sha256(identity)
|
||||
if (
|
||||
manifest.get("schema_version") != _E10_SCHEMA
|
||||
or manifest.get("pack_id") != expected_pack_id
|
||||
or manifest.get("identity_sha256") != identity_sha256
|
||||
or expected_pack_id != f"e10-lidar-pack-{identity_sha256}"
|
||||
or manifest.get("classification") != "private-recorded-sensor-replay-input"
|
||||
or manifest.get("ground_truth") is not False
|
||||
or identity.get("schema_version") != _E10_SCHEMA
|
||||
or identity.get("source_id") != M48_EXPECTED_E10_SOURCE_ID
|
||||
or identity.get("session_id") != M48_EXPECTED_SESSION_ID
|
||||
or identity.get("frame_count") != M48_EXPECTED_FRAME_COUNT
|
||||
or identity.get("source_start_frame_index") != 0
|
||||
or identity.get("source_end_frame_index") != M48_EXPECTED_FRAME_COUNT - 1
|
||||
or artifact.get("path") != _E10_ARTIFACT_NAME
|
||||
or artifact.get("media_type") != "application/x-npz"
|
||||
or artifact.get("sha256") != expected_artifact_sha256
|
||||
):
|
||||
raise M48RawEvidenceError("E10 pack identity changed")
|
||||
byte_length = artifact.get("byte_length")
|
||||
if not isinstance(byte_length, int) or isinstance(byte_length, bool) or byte_length < 1:
|
||||
raise M48RawEvidenceError("E10 artifact byte length is invalid")
|
||||
artifact_path = root / _E10_ARTIFACT_NAME
|
||||
if (
|
||||
artifact_path.is_symlink()
|
||||
or not artifact_path.is_file()
|
||||
or artifact_path.resolve(strict=True).parent != root
|
||||
or artifact_path.stat().st_size != byte_length
|
||||
or _file_sha256(artifact_path) != expected_artifact_sha256
|
||||
):
|
||||
raise M48RawEvidenceError("E10 artifact content changed")
|
||||
return artifact_path.resolve(strict=True)
|
||||
|
||||
|
||||
def _validate_m47_pack_binding(
|
||||
*,
|
||||
repository_root: Path,
|
||||
pack: M48ObjectQualityPack,
|
||||
threat_result: ThreatReplayResult,
|
||||
) -> None:
|
||||
identity = _mapping(pack.manifest.get("identity"), "M4.8 pack identity")
|
||||
source = _mapping(identity.get("source"), "M4.8 pack source")
|
||||
m47_id = source.get("m47_lab_result_id")
|
||||
m47_manifest_sha256 = source.get("m47_lab_manifest_sha256")
|
||||
if (
|
||||
source.get("source_id") != M48_EXPECTED_SOURCE_ID
|
||||
or source.get("source_session_id") != M48_EXPECTED_SESSION_ID
|
||||
or not isinstance(m47_id, str)
|
||||
or not isinstance(m47_manifest_sha256, str)
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 pack source binding changed")
|
||||
m47_root = repository_root / ".runtime/compute-experiments/m47/reference-graph-labs" / m47_id
|
||||
manifest_path = m47_root / "manifest.json"
|
||||
if (
|
||||
manifest_path.is_symlink()
|
||||
or not manifest_path.is_file()
|
||||
or _file_sha256(manifest_path) != m47_manifest_sha256
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 pack M4.7 manifest binding changed")
|
||||
try:
|
||||
m47 = read_m47_reference_graph_lab(m47_root)
|
||||
except (OSError, ValueError, M47ReferenceGraphLabError) as exc:
|
||||
raise M48RawEvidenceError("M4.8 pack M4.7 LAB is invalid") from exc
|
||||
m47_source = _mapping(m47.report.get("source"), "M4.7 source")
|
||||
threat_identity = _mapping(threat_result.manifest.get("identity"), "M4 threat identity")
|
||||
if (
|
||||
m47.manifest.get("accepted") is not True
|
||||
or m47_source.get("source_id") != M48_EXPECTED_SOURCE_ID
|
||||
or m47_source.get("source_session_id") != M48_EXPECTED_SESSION_ID
|
||||
or m47_source.get("visual_result_id") != threat_result.result_id
|
||||
or m47_source.get("threat_frames_sha256") != threat_identity.get("frames_sha256")
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 pack escaped its accepted M4.7 visual source")
|
||||
|
||||
|
||||
def _index_neutral_frame_references(
|
||||
pack: M48ObjectQualityPack,
|
||||
) -> dict[int, _PackFrameBinding]:
|
||||
index: dict[int, _PackFrameBinding] = {}
|
||||
for raw in pack.frame_references:
|
||||
row = _mapping(raw, "M4.8 neutral frame reference")
|
||||
sequence = row.get("sequence")
|
||||
source_time_ns = row.get("source_time_ns")
|
||||
clip_id = row.get("clip_id")
|
||||
if (
|
||||
not isinstance(sequence, int)
|
||||
or isinstance(sequence, bool)
|
||||
or not 1 <= sequence <= M48_EXPECTED_FRAME_COUNT
|
||||
or not isinstance(source_time_ns, int)
|
||||
or isinstance(source_time_ns, bool)
|
||||
or source_time_ns < 0
|
||||
or not isinstance(clip_id, str)
|
||||
or not clip_id
|
||||
or sequence in index
|
||||
):
|
||||
raise M48RawEvidenceError("M4.8 neutral frame references are invalid")
|
||||
index[sequence] = _PackFrameBinding(
|
||||
clip_id=clip_id,
|
||||
source_time_ns=source_time_ns,
|
||||
)
|
||||
if not index:
|
||||
raise M48RawEvidenceError("M4.8 neutral frame reference set is empty")
|
||||
return index
|
||||
|
||||
|
||||
def _strict_directory(path: Path, label: str) -> Path:
|
||||
candidate = path.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
raise M48RawEvidenceError(f"{label} must not be a symlink")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M48RawEvidenceError(f"{label} is unavailable") from exc
|
||||
if not resolved.is_dir():
|
||||
raise M48RawEvidenceError(f"{label} is not a directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _read_json(path: Path, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise M48RawEvidenceError(f"{label} is invalid") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48RawEvidenceError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _mapping(value: object, label: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise M48RawEvidenceError(f"{label} is invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"M48_EXPECTED_FRAME_COUNT",
|
||||
"M48_EXPECTED_SESSION_ID",
|
||||
"M48_EXPECTED_SOURCE_ID",
|
||||
"M48_EXPECTED_SOURCE_PACK_ID",
|
||||
"M48_EXPECTED_THREAT_RESULT_ID",
|
||||
"M48_RAW_SPATIAL_FRAME_SCHEMA",
|
||||
"M48RawEvidenceError",
|
||||
"M48RawEvidenceReader",
|
||||
]
|
||||
@@ -0,0 +1,711 @@
|
||||
"""Immutable M4.8 development regression over operator-added missed-object anchors.
|
||||
|
||||
The experiment deliberately stays inside M4.8 and reuses the frozen Worker 006
|
||||
prediction pack. It snapshots only operator-added tracklets from reviewed clips,
|
||||
compares the exact source frame against the already-frozen prediction row, and
|
||||
publishes a separate append-only result. The assisted correction is never called
|
||||
independent truth and the result grants no navigation or safety authority.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.laboratory.m48_object_quality import (
|
||||
M48ObjectQualityError,
|
||||
read_m48_object_quality_pack,
|
||||
)
|
||||
|
||||
M48_SMALL_STATIC_PROFILE_SCHEMA: Final = (
|
||||
"missioncore.m48-small-static-passage-regression-profile/v1"
|
||||
)
|
||||
M48_SMALL_STATIC_RESULT_SCHEMA: Final = (
|
||||
"missioncore.m48-small-static-passage-regression-result/v1"
|
||||
)
|
||||
M48_SMALL_STATIC_REPORT_SCHEMA: Final = (
|
||||
"missioncore.m48-small-static-passage-regression-report/v1"
|
||||
)
|
||||
M48_SMALL_STATIC_ANCHOR_SCHEMA: Final = (
|
||||
"missioncore.m48-assisted-missed-object-anchor/v1"
|
||||
)
|
||||
M48_SMALL_STATIC_COMPARISON_SCHEMA: Final = (
|
||||
"missioncore.m48-assisted-anchor-comparison/v1"
|
||||
)
|
||||
M48_SMALL_STATIC_PREFIX: Final = "m48-small-static-passage-regression-"
|
||||
|
||||
_CORRECTION_SCHEMA: Final = "missioncore.m48-assisted-object-correction-session/v1"
|
||||
_METHOD_SCHEMA: Final = "missioncore.laboratory-method/v1"
|
||||
_OBJECT_ID = re.compile(r"^object-[0-9]{2,}$")
|
||||
_AUTHORITY: Final = {
|
||||
"mode": "replay-simulated",
|
||||
"physical_live": False,
|
||||
"commands_enabled": False,
|
||||
"actuation_allowed": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class M48SmallStaticRegressionError(RuntimeError):
|
||||
"""The assisted development-regression source or result is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M48SmallStaticRegressionResult:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
manifest: dict[str, Any]
|
||||
report: dict[str, Any]
|
||||
anchors: tuple[dict[str, Any], ...]
|
||||
comparisons: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
def build_m48_small_static_passage_regression(
|
||||
*,
|
||||
pack_root: Path,
|
||||
correction_session_path: Path,
|
||||
profile_path: Path,
|
||||
output_root: Path,
|
||||
run_created_at_utc: str | None = None,
|
||||
) -> M48SmallStaticRegressionResult:
|
||||
"""Publish one append-only M4.8R development baseline without mutating inputs."""
|
||||
|
||||
try:
|
||||
pack = read_m48_object_quality_pack(pack_root)
|
||||
except M48ObjectQualityError as exc:
|
||||
raise M48SmallStaticRegressionError("M4.8 frozen prediction pack is invalid") from exc
|
||||
profile_bytes, profile = _read_profile(profile_path)
|
||||
correction_bytes, correction = _read_correction(correction_session_path, pack.result_id)
|
||||
anchors = _assisted_anchors(correction)
|
||||
if len(anchors) < int(profile["minimum_anchor_count"]):
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted anchor set is too small")
|
||||
|
||||
prediction_rows: dict[tuple[str, int], dict[str, Any]] = {}
|
||||
for row in pack.predictions:
|
||||
clip_id = row.get("clip_id")
|
||||
sequence = row.get("sequence")
|
||||
if not isinstance(clip_id, str) or not _integer(sequence):
|
||||
raise M48SmallStaticRegressionError("M4.8 frozen prediction binding is invalid")
|
||||
key = (clip_id, int(sequence))
|
||||
if key in prediction_rows:
|
||||
raise M48SmallStaticRegressionError("M4.8 frozen prediction binding collided")
|
||||
prediction_rows[key] = row
|
||||
|
||||
threshold = float(profile["extent_iou_threshold"])
|
||||
comparisons = tuple(
|
||||
_compare_anchor(anchor, prediction_rows, threshold)
|
||||
for anchor in anchors
|
||||
)
|
||||
recalled = sum(bool(row["matched_at_threshold"]) for row in comparisons)
|
||||
recall = recalled / len(comparisons)
|
||||
passage_count = sum(bool(row["requires_avoidance_or_clearance"]) for row in anchors)
|
||||
clip_count = len({str(row["clip_id"]) for row in anchors})
|
||||
target = float(profile["minimum_assisted_anchor_recall"])
|
||||
accepted = recall >= target
|
||||
created_at = _utc_timestamp(run_created_at_utc or datetime.now(UTC).isoformat())
|
||||
correction_sha256 = hashlib.sha256(correction_bytes).hexdigest()
|
||||
profile_sha256 = hashlib.sha256(profile_bytes).hexdigest()
|
||||
producer_sha256 = _file_sha256(Path(__file__).resolve())
|
||||
pack_identity = _object(pack.manifest.get("identity"), "M4.8 pack identity")
|
||||
freeze = _object(pack_identity.get("freeze"), "M4.8 pack freeze")
|
||||
|
||||
identity: dict[str, Any] = {
|
||||
"schema_version": M48_SMALL_STATIC_RESULT_SCHEMA,
|
||||
"human_lab_id": profile["human_lab_id"],
|
||||
"run_label": profile["run_label"],
|
||||
"run_created_at_utc": created_at,
|
||||
"pipeline_id": profile["pipeline_id"],
|
||||
"experiment_id": profile["experiment_id"],
|
||||
"profile_id": profile["profile_id"],
|
||||
"profile_sha256": profile_sha256,
|
||||
"producer_sha256": producer_sha256,
|
||||
"source": {
|
||||
"source_id": _object(
|
||||
pack_identity.get("source"), "M4.8 source"
|
||||
).get("source_id"),
|
||||
"source_session_id": _object(
|
||||
pack_identity.get("source"), "M4.8 source"
|
||||
).get("source_session_id"),
|
||||
"pack_id": pack.result_id,
|
||||
"pack_identity_sha256": pack.manifest["identity_sha256"],
|
||||
"prediction_rows_sha256": freeze.get("prediction_rows_sha256"),
|
||||
"correction_session_id": correction["session_id"],
|
||||
"correction_revision": correction["revision"],
|
||||
"correction_updated_at_utc": correction["updated_at_utc"],
|
||||
"correction_document_sha256": correction_sha256,
|
||||
"correction_independent_truth": False,
|
||||
},
|
||||
"selection": {
|
||||
"anchor_selection": profile["anchor_selection"],
|
||||
"assisted_tracklet_count": len({(row["clip_id"], row["object_id"]) for row in anchors}),
|
||||
"anchor_count": len(anchors),
|
||||
"clip_count": clip_count,
|
||||
"requires_avoidance_or_clearance_count": passage_count,
|
||||
},
|
||||
"authority": dict(_AUTHORITY),
|
||||
}
|
||||
identity_sha256 = _canonical_sha256(identity)
|
||||
result_id = f"{M48_SMALL_STATIC_PREFIX}{identity_sha256}"
|
||||
method = {
|
||||
"schema_version": _METHOD_SCHEMA,
|
||||
"completeness": "complete",
|
||||
"execution_class": "deterministic",
|
||||
"pipeline_id": profile["pipeline_id"],
|
||||
"components": [
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "M4.8 frozen Worker 006 predictions",
|
||||
"version": pack.result_id,
|
||||
"role": "immutable candidate rows from the current M4.8 pipeline",
|
||||
"identity_sha256": freeze.get("prediction_rows_sha256"),
|
||||
},
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "operator-added missed-object anchors",
|
||||
"version": f"{correction['session_id']}:revision-{correction['revision']}",
|
||||
"role": "assisted development regression seed; not independent truth",
|
||||
"identity_sha256": correction_sha256,
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": "exact-frame class-free IoU comparator",
|
||||
"version": profile["profile_id"],
|
||||
"role": "diagnostic detection recall over operator-added anchors",
|
||||
"identity_sha256": producer_sha256,
|
||||
},
|
||||
],
|
||||
}
|
||||
metrics = {
|
||||
"assisted_anchor_count": len(comparisons),
|
||||
"assisted_tracklet_count": len({(row["clip_id"], row["object_id"]) for row in anchors}),
|
||||
"anchor_clip_count": clip_count,
|
||||
"requires_avoidance_or_clearance_count": passage_count,
|
||||
"worker_recalled_anchor_count": recalled,
|
||||
"worker_missed_anchor_count": len(comparisons) - recalled,
|
||||
"assisted_anchor_recall": recall,
|
||||
"extent_iou_threshold": threshold,
|
||||
"minimum_assisted_anchor_recall": target,
|
||||
}
|
||||
gates = {
|
||||
"anchor_set_non_empty": len(comparisons) >= int(profile["minimum_anchor_count"]),
|
||||
"development_anchor_recall_target": accepted,
|
||||
"independent_truth_available": False,
|
||||
}
|
||||
decision = {
|
||||
"state": (
|
||||
"accepted-development-regression-baseline"
|
||||
if accepted
|
||||
else "failed-development-regression-baseline"
|
||||
),
|
||||
"summary": (
|
||||
f"Worker 006 matched {recalled}/{len(comparisons)} exact-frame assisted anchors "
|
||||
f"at IoU >= {threshold:.2f}."
|
||||
),
|
||||
"next_action": (
|
||||
"Keep the pipeline contract fixed, change only the perception experiment, "
|
||||
"and publish another immutable M4.8R run against this frozen seed."
|
||||
),
|
||||
}
|
||||
limitations = [
|
||||
"The anchors come from candidate-visible operator correction and are not "
|
||||
"independent truth.",
|
||||
"The seed is intentionally biased toward objects the current Worker 006 output missed.",
|
||||
"A camera rectangle is evidence of a missed visible object, not a measured 3D collider.",
|
||||
"No physical-live, navigation, command, actuation or collision-safety "
|
||||
"authority is granted.",
|
||||
]
|
||||
report = {
|
||||
"schema_version": M48_SMALL_STATIC_REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"source": identity["source"],
|
||||
"configuration": {
|
||||
**profile,
|
||||
"profile_sha256": profile_sha256,
|
||||
},
|
||||
"method": method,
|
||||
"execution": {
|
||||
"comparison_node": "mission-core-local-control-plane",
|
||||
"source_worker_id": "006",
|
||||
"frozen_prediction_rows_sha256": freeze.get("prediction_rows_sha256"),
|
||||
"determinism": "exact canonical JSON + exact-frame IoU; no inference rerun",
|
||||
},
|
||||
"metrics": metrics,
|
||||
"gates": gates,
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
"authority": dict(_AUTHORITY),
|
||||
"visual_review": {
|
||||
"viewer": "missioncore.laboratory-recorded-clip-viewer/v1",
|
||||
"case_count": len(comparisons),
|
||||
"camera_anchor_and_worker_boxes": True,
|
||||
"camera_3d_plan_shared_clock": True,
|
||||
},
|
||||
}
|
||||
destination = output_root.expanduser().absolute() / result_id
|
||||
_publish_result(
|
||||
destination=destination,
|
||||
identity=identity,
|
||||
created_at_utc=created_at,
|
||||
accepted=accepted,
|
||||
report=report,
|
||||
anchors=anchors,
|
||||
comparisons=comparisons,
|
||||
)
|
||||
return read_m48_small_static_passage_regression(destination)
|
||||
|
||||
|
||||
def read_m48_small_static_passage_regression(
|
||||
root: Path,
|
||||
) -> M48SmallStaticRegressionResult:
|
||||
candidate = root.expanduser().absolute()
|
||||
if candidate.is_symlink():
|
||||
raise M48SmallStaticRegressionError("M4.8 regression result must not be a symlink")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression result is unavailable") from exc
|
||||
if not resolved.is_dir() or not resolved.name.startswith(M48_SMALL_STATIC_PREFIX):
|
||||
raise M48SmallStaticRegressionError("M4.8 regression result path is invalid")
|
||||
manifest = _read_json(resolved / "manifest.json", maximum=1024 * 1024)
|
||||
identity = _object(manifest.get("identity"), "M4.8 regression identity")
|
||||
identity_sha256 = _canonical_sha256(identity)
|
||||
if (
|
||||
manifest.get("schema_version") != M48_SMALL_STATIC_RESULT_SCHEMA
|
||||
or manifest.get("result_id") != resolved.name
|
||||
or manifest.get("identity_sha256") != identity_sha256
|
||||
or resolved.name != f"{M48_SMALL_STATIC_PREFIX}{identity_sha256}"
|
||||
or manifest.get("ground_truth") is not False
|
||||
or manifest.get("authority") != _AUTHORITY
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 regression identity changed")
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list) or len(artifacts) != 3:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression artifact inventory changed")
|
||||
by_path: dict[str, dict[str, Any]] = {}
|
||||
for raw in artifacts:
|
||||
descriptor = _object(raw, "M4.8 regression artifact")
|
||||
path_name = descriptor.get("path")
|
||||
if not isinstance(path_name, str) or path_name not in {
|
||||
"anchors.jsonl", "comparisons.jsonl", "report.json"
|
||||
} or path_name in by_path:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression artifact path changed")
|
||||
path = resolved / path_name
|
||||
if (
|
||||
path.is_symlink()
|
||||
or not path.is_file()
|
||||
or descriptor.get("byte_length") != path.stat().st_size
|
||||
or descriptor.get("sha256") != _file_sha256(path)
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 regression artifact proof changed")
|
||||
by_path[path_name] = descriptor
|
||||
report = _read_json(resolved / "report.json", maximum=1024 * 1024)
|
||||
anchors = tuple(_read_jsonl(resolved / "anchors.jsonl"))
|
||||
comparisons = tuple(_read_jsonl(resolved / "comparisons.jsonl"))
|
||||
if (
|
||||
report.get("schema_version") != M48_SMALL_STATIC_REPORT_SCHEMA
|
||||
or report.get("result_id") != resolved.name
|
||||
or len(anchors) != len(comparisons)
|
||||
or any(row.get("schema_version") != M48_SMALL_STATIC_ANCHOR_SCHEMA for row in anchors)
|
||||
or any(
|
||||
row.get("schema_version") != M48_SMALL_STATIC_COMPARISON_SCHEMA
|
||||
for row in comparisons
|
||||
)
|
||||
or [row.get("anchor_id") for row in anchors]
|
||||
!= [row.get("anchor_id") for row in comparisons]
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 regression content changed")
|
||||
return M48SmallStaticRegressionResult(
|
||||
result_id=resolved.name,
|
||||
result_root=resolved,
|
||||
manifest=manifest,
|
||||
report=report,
|
||||
anchors=anchors,
|
||||
comparisons=comparisons,
|
||||
)
|
||||
|
||||
|
||||
def _read_profile(path: Path) -> tuple[bytes, dict[str, Any]]:
|
||||
encoded, profile = _read_json_bytes(path, maximum=64 * 1024, label="M4.8 regression profile")
|
||||
expected = {
|
||||
"schema_version",
|
||||
"profile_id",
|
||||
"pipeline_id",
|
||||
"experiment_id",
|
||||
"human_lab_id",
|
||||
"run_label",
|
||||
"anchor_selection",
|
||||
"extent_iou_threshold",
|
||||
"minimum_assisted_anchor_recall",
|
||||
"minimum_anchor_count",
|
||||
"independent_truth",
|
||||
}
|
||||
if set(profile) != expected or profile.get("schema_version") != M48_SMALL_STATIC_PROFILE_SCHEMA:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression profile contract changed")
|
||||
if (
|
||||
profile.get("human_lab_id") != "M4.8"
|
||||
or profile.get("anchor_selection") != "operator-added-tracklets-in-reviewed-clips/v1"
|
||||
or profile.get("independent_truth") is not False
|
||||
or not _rate(profile.get("extent_iou_threshold"))
|
||||
or not _rate(profile.get("minimum_assisted_anchor_recall"))
|
||||
or not _integer(profile.get("minimum_anchor_count"))
|
||||
or int(profile["minimum_anchor_count"]) < 1
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 regression profile is invalid")
|
||||
for key in ("profile_id", "pipeline_id", "experiment_id", "run_label"):
|
||||
if not isinstance(profile.get(key), str) or not str(profile[key]).strip():
|
||||
raise M48SmallStaticRegressionError("M4.8 regression profile identity is invalid")
|
||||
return encoded, profile
|
||||
|
||||
|
||||
def _read_correction(path: Path, pack_id: str) -> tuple[bytes, dict[str, Any]]:
|
||||
encoded, correction = _read_json_bytes(
|
||||
path,
|
||||
maximum=16 * 1024 * 1024,
|
||||
label="M4.8 correction snapshot",
|
||||
)
|
||||
assistance = _object(correction.get("assistance"), "M4.8 correction assistance")
|
||||
if (
|
||||
correction.get("schema_version") != _CORRECTION_SCHEMA
|
||||
or correction.get("pack_id") != pack_id
|
||||
or correction.get("state") not in {"saved", "frozen"}
|
||||
or not _integer(correction.get("revision"))
|
||||
or int(correction["revision"]) < 1
|
||||
or not isinstance(correction.get("session_id"), str)
|
||||
or not isinstance(correction.get("updated_at_utc"), str)
|
||||
or assistance.get("candidate_predictions_seen") is not True
|
||||
or assistance.get("independent_truth_eligible") is not False
|
||||
or correction.get("authority") != _AUTHORITY
|
||||
or not isinstance(correction.get("clips"), list)
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 correction snapshot is invalid")
|
||||
return encoded, correction
|
||||
|
||||
|
||||
def _assisted_anchors(correction: dict[str, Any]) -> tuple[dict[str, Any], ...]:
|
||||
anchors: list[dict[str, Any]] = []
|
||||
for clip_raw in correction["clips"]:
|
||||
clip = _object(clip_raw, "M4.8 correction clip")
|
||||
if clip.get("review_state") != "reviewed":
|
||||
continue
|
||||
clip_id = clip.get("clip_id")
|
||||
tracklets = clip.get("tracklets")
|
||||
if not isinstance(clip_id, str) or not isinstance(tracklets, list):
|
||||
raise M48SmallStaticRegressionError("M4.8 correction clip is invalid")
|
||||
for tracklet_raw in tracklets:
|
||||
tracklet = _object(tracklet_raw, "M4.8 correction tracklet")
|
||||
object_id = tracklet.get("object_id")
|
||||
if not isinstance(object_id, str) or _OBJECT_ID.fullmatch(object_id) is None:
|
||||
continue
|
||||
keyframes = tracklet.get("keyframes")
|
||||
if not isinstance(keyframes, list) or not keyframes:
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted tracklet has no keyframes")
|
||||
for keyframe_raw in keyframes:
|
||||
keyframe = _object(keyframe_raw, "M4.8 correction keyframe")
|
||||
sequence = keyframe.get("sequence")
|
||||
extent = _extent(keyframe.get("extent_xyxy"))
|
||||
if not _integer(sequence):
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted anchor sequence is invalid")
|
||||
state = _state_for_sequence(tracklet, int(sequence))
|
||||
anchor_identity = {
|
||||
"clip_id": clip_id,
|
||||
"object_id": object_id,
|
||||
"sequence": int(sequence),
|
||||
"extent_xyxy": extent,
|
||||
}
|
||||
anchors.append({
|
||||
"schema_version": M48_SMALL_STATIC_ANCHOR_SCHEMA,
|
||||
"anchor_id": "anchor-" + _canonical_sha256(anchor_identity)[:24],
|
||||
**anchor_identity,
|
||||
"visibility": keyframe.get("visibility"),
|
||||
"geometry_association": state.get("geometry_association"),
|
||||
"freshness": state.get("freshness"),
|
||||
"motion": state.get("motion"),
|
||||
"threat": state.get("threat"),
|
||||
"requires_avoidance_or_clearance": bool(
|
||||
state.get("critical_corridor_obstacle")
|
||||
),
|
||||
"authority": "operator-assisted-development-anchor-not-truth",
|
||||
})
|
||||
anchors.sort(key=lambda row: (str(row["clip_id"]), int(row["sequence"]), str(row["object_id"])))
|
||||
if len({str(row["anchor_id"]) for row in anchors}) != len(anchors):
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted anchor identity collided")
|
||||
return tuple(anchors)
|
||||
|
||||
|
||||
def _state_for_sequence(tracklet: dict[str, Any], sequence: int) -> dict[str, Any]:
|
||||
segments = tracklet.get("state_segments")
|
||||
if not isinstance(segments, list):
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted state segments are invalid")
|
||||
matches = [
|
||||
_object(row, "M4.8 assisted state segment")
|
||||
for row in segments
|
||||
if isinstance(row, dict)
|
||||
and _integer(row.get("start_sequence"))
|
||||
and _integer(row.get("end_sequence"))
|
||||
and int(row["start_sequence"]) <= sequence <= int(row["end_sequence"])
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted anchor state is ambiguous")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _compare_anchor(
|
||||
anchor: dict[str, Any],
|
||||
prediction_rows: dict[tuple[str, int], dict[str, Any]],
|
||||
threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
key = (str(anchor["clip_id"]), int(anchor["sequence"]))
|
||||
row = prediction_rows.get(key)
|
||||
if row is None or row.get("terminal_outcome") != "delivered":
|
||||
raise M48SmallStaticRegressionError("M4.8 assisted anchor lacks delivered prediction row")
|
||||
objects = row.get("objects")
|
||||
if not isinstance(objects, list):
|
||||
raise M48SmallStaticRegressionError("M4.8 prediction objects are invalid")
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for raw in objects:
|
||||
item = _object(raw, "M4.8 prediction object")
|
||||
normalized.append({
|
||||
"prediction_id": item.get("prediction_id"),
|
||||
"extent_xyxy": _extent(item.get("extent_xyxy")),
|
||||
"geometry_association": item.get("geometry_association"),
|
||||
"freshness": item.get("freshness"),
|
||||
"motion": item.get("motion"),
|
||||
"threat": item.get("threat"),
|
||||
})
|
||||
ranked = sorted(
|
||||
((_iou(anchor["extent_xyxy"], item["extent_xyxy"]), item) for item in normalized),
|
||||
key=lambda pair: (pair[0], str(pair[1].get("prediction_id"))),
|
||||
reverse=True,
|
||||
)
|
||||
best_iou, best = ranked[0] if ranked else (0.0, None)
|
||||
return {
|
||||
"schema_version": M48_SMALL_STATIC_COMPARISON_SCHEMA,
|
||||
"anchor_id": anchor["anchor_id"],
|
||||
"clip_id": anchor["clip_id"],
|
||||
"sequence": anchor["sequence"],
|
||||
"source_time_ns": row.get("source_time_ns"),
|
||||
"anchor_extent_xyxy": anchor["extent_xyxy"],
|
||||
"requires_avoidance_or_clearance": anchor["requires_avoidance_or_clearance"],
|
||||
"worker_candidate_count": len(normalized),
|
||||
"worker_objects": normalized,
|
||||
"best_prediction_id": best.get("prediction_id") if best else None,
|
||||
"best_iou": best_iou,
|
||||
"extent_iou_threshold": threshold,
|
||||
"matched_at_threshold": best_iou >= threshold,
|
||||
"outcome": "recalled" if best_iou >= threshold else "missed-assisted-anchor",
|
||||
}
|
||||
|
||||
|
||||
def _publish_result(
|
||||
*,
|
||||
destination: Path,
|
||||
identity: dict[str, Any],
|
||||
created_at_utc: str,
|
||||
accepted: bool,
|
||||
report: dict[str, Any],
|
||||
anchors: tuple[dict[str, Any], ...],
|
||||
comparisons: tuple[dict[str, Any], ...],
|
||||
) -> None:
|
||||
parent = destination.parent
|
||||
if parent.is_symlink():
|
||||
raise M48SmallStaticRegressionError("M4.8 regression output root must not be a symlink")
|
||||
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if not parent.is_dir():
|
||||
raise M48SmallStaticRegressionError("M4.8 regression output root is invalid")
|
||||
staging = parent / f".{destination.name}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
_write_json(staging / "report.json", report)
|
||||
_write_jsonl(staging / "anchors.jsonl", anchors)
|
||||
_write_jsonl(staging / "comparisons.jsonl", comparisons)
|
||||
artifacts = [
|
||||
_artifact(
|
||||
staging / "anchors.jsonl",
|
||||
"assisted-regression-anchors",
|
||||
M48_SMALL_STATIC_ANCHOR_SCHEMA,
|
||||
),
|
||||
_artifact(
|
||||
staging / "comparisons.jsonl",
|
||||
"exact-frame-worker-comparisons",
|
||||
M48_SMALL_STATIC_COMPARISON_SCHEMA,
|
||||
),
|
||||
_artifact(
|
||||
staging / "report.json",
|
||||
"m48-small-static-regression-report",
|
||||
M48_SMALL_STATIC_REPORT_SCHEMA,
|
||||
),
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": M48_SMALL_STATIC_RESULT_SCHEMA,
|
||||
"result_id": destination.name,
|
||||
"identity_sha256": _canonical_sha256(identity),
|
||||
"identity": identity,
|
||||
"created_at_utc": created_at_utc,
|
||||
"accepted": accepted,
|
||||
"ground_truth": False,
|
||||
"authority": dict(_AUTHORITY),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(staging / "manifest.json", manifest)
|
||||
if destination.exists():
|
||||
existing = {
|
||||
path.name: _file_sha256(path)
|
||||
for path in destination.iterdir()
|
||||
if path.is_file()
|
||||
}
|
||||
proposed = {
|
||||
path.name: _file_sha256(path)
|
||||
for path in staging.iterdir()
|
||||
if path.is_file()
|
||||
}
|
||||
if existing != proposed:
|
||||
raise M48SmallStaticRegressionError("immutable M4.8 regression identity collided")
|
||||
shutil.rmtree(staging)
|
||||
return
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def _artifact(path: Path, role: str, schema_version: str) -> dict[str, object]:
|
||||
return {
|
||||
"path": path.name,
|
||||
"role": role,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _file_sha256(path),
|
||||
"schema_version": schema_version,
|
||||
"media_type": "application/x-ndjson" if path.suffix == ".jsonl" else "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _read_json_bytes(path: Path, *, maximum: int, label: str) -> tuple[bytes, dict[str, Any]]:
|
||||
candidate = path.expanduser().absolute()
|
||||
if candidate.is_symlink() or not candidate.is_file() or candidate.stat().st_size > maximum:
|
||||
raise M48SmallStaticRegressionError(f"{label} is unavailable")
|
||||
try:
|
||||
encoded = candidate.read_bytes()
|
||||
value = json.loads(encoded)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise M48SmallStaticRegressionError(f"{label} is unreadable") from exc
|
||||
return encoded, _object(value, label)
|
||||
|
||||
|
||||
def _read_json(path: Path, *, maximum: int) -> dict[str, Any]:
|
||||
return _read_json_bytes(path, maximum=maximum, label=path.name)[1]
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 * 1024 * 1024:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression rows are unavailable")
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
for line in stream:
|
||||
if line.strip():
|
||||
rows.append(_object(json.loads(line), "M4.8 regression row"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise M48SmallStaticRegressionError("M4.8 regression rows are unreadable") from exc
|
||||
return rows
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_bytes(_canonical_json(value) + b"\n")
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: tuple[dict[str, Any], ...]) -> None:
|
||||
path.write_bytes(b"".join(_canonical_json(row) + b"\n" for row in rows))
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||
raise M48SmallStaticRegressionError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _integer(value: object) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool)
|
||||
|
||||
|
||||
def _rate(value: object) -> bool:
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and 0.0 < float(value) <= 1.0
|
||||
)
|
||||
|
||||
|
||||
def _extent(value: object) -> list[float]:
|
||||
if (
|
||||
not isinstance(value, list)
|
||||
or len(value) != 4
|
||||
or any(not isinstance(item, (int, float)) or isinstance(item, bool) for item in value)
|
||||
):
|
||||
raise M48SmallStaticRegressionError("M4.8 extent is invalid")
|
||||
extent = [float(item) for item in value]
|
||||
if not (0.0 <= extent[0] < extent[2] <= 1.0 and 0.0 <= extent[1] < extent[3] <= 1.0):
|
||||
raise M48SmallStaticRegressionError("M4.8 extent is outside the camera plane")
|
||||
return extent
|
||||
|
||||
|
||||
def _iou(left: list[float], right: list[float]) -> float:
|
||||
x1 = max(left[0], right[0])
|
||||
y1 = max(left[1], right[1])
|
||||
x2 = min(left[2], right[2])
|
||||
y2 = min(left[3], right[3])
|
||||
intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1)
|
||||
left_area = (left[2] - left[0]) * (left[3] - left[1])
|
||||
right_area = (right[2] - right[0]) * (right[3] - right[1])
|
||||
union = left_area + right_area - intersection
|
||||
return intersection / union if union > 0.0 else 0.0
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _canonical_sha256(value: object) -> str:
|
||||
return hashlib.sha256(_canonical_json(value)).hexdigest()
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _utc_timestamp(value: object) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise M48SmallStaticRegressionError("M4.8 run creation time is invalid")
|
||||
text = value.strip()
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise M48SmallStaticRegressionError("M4.8 run creation time is invalid") from exc
|
||||
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
||||
raise M48SmallStaticRegressionError("M4.8 run creation time must be UTC")
|
||||
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"M48_SMALL_STATIC_RESULT_SCHEMA",
|
||||
"M48SmallStaticRegressionError",
|
||||
"M48SmallStaticRegressionResult",
|
||||
"build_m48_small_static_passage_regression",
|
||||
"read_m48_small_static_passage_regression",
|
||||
]
|
||||
@@ -56,6 +56,7 @@ from k1link.compute.e40_perception_product_gate import (
|
||||
read_e40_perception_product_gate,
|
||||
)
|
||||
from k1link.laboratory import LaboratoryEvidenceDefinition, LaboratoryEvidenceRegistry
|
||||
from k1link.laboratory.evidence_registry import LaboratoryEvidenceVariant
|
||||
from k1link.web.l3_pointpillars_visual_api import latest_l3_visual_identity
|
||||
from k1link.web.l31_pointpillars_ravnoves_api import latest_l31_identity
|
||||
from k1link.web.l32_pointpillars_camera_review_api import latest_l32_identity
|
||||
@@ -259,7 +260,10 @@ def _advanced_index(
|
||||
specs: tuple[_AdvancedIndexSpec, ...],
|
||||
) -> dict[str, object]:
|
||||
items: list[dict[str, object]] = []
|
||||
selected_work_ids: set[str] = set()
|
||||
for work_id, provider, pattern, document_name, schema_version in specs:
|
||||
if work_id in selected_work_ids:
|
||||
continue
|
||||
root = _configured_root(provider)
|
||||
if root is None:
|
||||
continue
|
||||
@@ -273,6 +277,7 @@ def _advanced_index(
|
||||
schema_version=schema_version,
|
||||
)
|
||||
)
|
||||
selected_work_ids.add(work_id)
|
||||
break
|
||||
except (json.JSONDecodeError, OSError, TypeError, ValueError):
|
||||
continue
|
||||
@@ -290,17 +295,18 @@ def _registry_index_specs(
|
||||
return tuple(
|
||||
(
|
||||
definition.work_id,
|
||||
_evidence_root_provider(definition, runtime_root_provider),
|
||||
definition.result_id_pattern,
|
||||
definition.document_name,
|
||||
definition.result_schema_version,
|
||||
_evidence_root_provider(variant, runtime_root_provider),
|
||||
variant.result_id_pattern,
|
||||
variant.document_name,
|
||||
variant.result_schema_version,
|
||||
)
|
||||
for definition in registry.definitions
|
||||
for variant in reversed(definition.evidence_variants)
|
||||
)
|
||||
|
||||
|
||||
def _evidence_root_provider(
|
||||
definition: LaboratoryEvidenceDefinition,
|
||||
definition: LaboratoryEvidenceDefinition | LaboratoryEvidenceVariant,
|
||||
runtime_root_provider: RootProvider,
|
||||
) -> RootProvider:
|
||||
def result_root_provider() -> Path | None:
|
||||
|
||||
@@ -23,15 +23,23 @@ from k1link.compute import (
|
||||
RecordedPerceptionOverlayMux,
|
||||
RecordedPerceptionOverlayStore,
|
||||
)
|
||||
from k1link.compute.pipeline_telemetry import JsonlPipelineTelemetrySink
|
||||
from k1link.laboratory import (
|
||||
LaboratoryEvidenceRegistry,
|
||||
LaboratoryEvidenceReportService,
|
||||
LaboratoryExecutionRegistry,
|
||||
LaboratoryRunner,
|
||||
LaboratoryValueReviewRegistry,
|
||||
)
|
||||
from k1link.laboratory.m48_raw_evidence import (
|
||||
M48_EXPECTED_THREAT_RESULT_ID,
|
||||
M48RawEvidenceError,
|
||||
M48RawEvidenceReader,
|
||||
)
|
||||
from k1link.sessions import (
|
||||
MaterializedRecording,
|
||||
RecordedCameraFrameService,
|
||||
RecordedCameraPlaybackSource,
|
||||
RecordedMediaInspector,
|
||||
RecordedMediaManifest,
|
||||
RecordingPreparationQueueFull,
|
||||
@@ -116,6 +124,7 @@ from k1link.web.laboratory_report_api import build_laboratory_report_router
|
||||
from k1link.web.lidar_api import build_lidar_router
|
||||
from k1link.web.lidar_local_surface_service import K1LocalSurfaceReadService
|
||||
from k1link.web.m4_threat_replay_api import build_m4_threat_replay_router
|
||||
from k1link.web.m48_object_quality_api import build_m48_object_quality_router
|
||||
from k1link.web.map_api import (
|
||||
MapGatewayConfiguration,
|
||||
MapGatewayProxy,
|
||||
@@ -151,6 +160,13 @@ LABORATORY_EXECUTION_REGISTRY = LaboratoryExecutionRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "laboratory-execution.json",
|
||||
LABORATORY_EVIDENCE_REGISTRY,
|
||||
)
|
||||
LABORATORY_RUNNER = LaboratoryRunner(
|
||||
registry=LABORATORY_EXECUTION_REGISTRY,
|
||||
evidence_registry=LABORATORY_EVIDENCE_REGISTRY,
|
||||
sink=JsonlPipelineTelemetrySink(
|
||||
REPOSITORY_ROOT / ".runtime" / "telemetry" / "laboratory-runs.jsonl"
|
||||
),
|
||||
)
|
||||
LABORATORY_VALUE_REVIEW_REGISTRY = LaboratoryValueReviewRegistry.from_file(
|
||||
REPOSITORY_ROOT / "config" / "laboratory-value-review.json"
|
||||
)
|
||||
@@ -204,6 +220,20 @@ session_recorded_camera_frame_service = (
|
||||
if _ffmpeg is not None
|
||||
else None
|
||||
)
|
||||
try:
|
||||
m48_raw_evidence_reader: M48RawEvidenceReader | None = M48RawEvidenceReader.from_repository(
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
threat_result_root=(
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m4"
|
||||
/ "replay-threat"
|
||||
/ M48_EXPECTED_THREAT_RESULT_ID
|
||||
),
|
||||
)
|
||||
except (M48RawEvidenceError, OSError, ValueError):
|
||||
m48_raw_evidence_reader = None
|
||||
session_legacy_perception_overlay_store = (
|
||||
RecordedPerceptionOverlayStore(
|
||||
jobs_root=REPOSITORY_ROOT / ".runtime" / "compute-jobs",
|
||||
@@ -295,6 +325,20 @@ session_recording_preparation_manager = SessionRecordingPreparationManager(
|
||||
)
|
||||
|
||||
|
||||
def _m48_recorded_camera_playback_source(
|
||||
session_id: str,
|
||||
) -> RecordedCameraPlaybackSource:
|
||||
"""Publish the durable replay package before exposing its manifest URL."""
|
||||
|
||||
if session_recorded_camera_frame_service is None:
|
||||
raise RuntimeError("recorded camera playback is unavailable")
|
||||
command = session_store.prepare_replay(session_id, speed=1.0, loop=False)
|
||||
snapshot = session_recording_preparation_manager.restore_published(command)
|
||||
if snapshot is None or snapshot.state != "ready" or snapshot.recorded_media is None:
|
||||
raise RuntimeError("recorded camera playback package is not published")
|
||||
return session_recorded_camera_frame_service.playback_source(session_id)
|
||||
|
||||
|
||||
def refresh_observation_catalog() -> tuple[str, ...]:
|
||||
"""Discover completed or recoverable local evidence without copying payloads."""
|
||||
|
||||
@@ -859,6 +903,47 @@ app.include_router(
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m48_object_quality_router(
|
||||
pack_root_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-packs"
|
||||
),
|
||||
workflow_root_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "laboratory-annotations" / "m48-object-quality"
|
||||
),
|
||||
truth_root_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-truth-seals"
|
||||
),
|
||||
result_root_provider=lambda: (
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m48" / "object-quality-results"
|
||||
),
|
||||
small_static_result_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "m48"
|
||||
/ "small-static-passage-regression-results"
|
||||
),
|
||||
camera_frame_provider=(
|
||||
session_recorded_camera_frame_service.extract
|
||||
if session_recorded_camera_frame_service is not None
|
||||
else None
|
||||
),
|
||||
camera_playback_provider=(
|
||||
_m48_recorded_camera_playback_source
|
||||
if session_recorded_camera_frame_service is not None
|
||||
else None
|
||||
),
|
||||
spatial_evidence_provider=m48_raw_evidence_reader,
|
||||
evaluation_runner=LABORATORY_RUNNER,
|
||||
evaluation_receipt_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "laboratory-run-receipts"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_e47_semantic_slam_router(
|
||||
root_provider=lambda: (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user