feat(perception): audit E36 second-source eligibility
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,488 @@
|
||||
"""Fail-closed E36 catalog admission for a second mounted real source.
|
||||
|
||||
The audit is deliberately separate from an E36 replay. It decides whether the
|
||||
catalog contains a source on which the frozen E32--E35 profile may be run
|
||||
without inventing calibration or mount provenance. A blocked audit is a real,
|
||||
content-addressed engineering result, but it is not a LAB publication.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.artifacts import utc_now_iso
|
||||
|
||||
E36_PROFILE_SCHEMA: Final = "missioncore.e36-source-eligibility-profile/v1"
|
||||
E36_INVENTORY_SCHEMA: Final = "missioncore.e36-source-catalog-inventory/v1"
|
||||
E36_AUDIT_SCHEMA: Final = "missioncore.e36-source-catalog-audit/v1"
|
||||
E36_REPORT_SCHEMA: Final = "missioncore.e36-source-catalog-audit-report/v1"
|
||||
|
||||
E36_REPORT_NAME: Final = "audit-report.json"
|
||||
E36_MANIFEST_NAME: Final = "manifest.json"
|
||||
|
||||
_RESULT_ID = re.compile(r"^e36-source-catalog-audit-[a-f0-9]{64}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_SESSION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
|
||||
|
||||
|
||||
class E36SourceCatalogAuditError(RuntimeError):
|
||||
"""The E36 profile, inventory or immutable result is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class E36SourceCatalogAudit:
|
||||
result_root: Path
|
||||
result_id: str
|
||||
manifest: dict[str, Any]
|
||||
report: dict[str, Any]
|
||||
|
||||
@property
|
||||
def eligible_source_ids(self) -> tuple[str, ...]:
|
||||
candidates = _array(self.report.get("candidates"), "E36 candidates")
|
||||
return tuple(
|
||||
_required_string(item, "session_id")
|
||||
for item in candidates
|
||||
if isinstance(item, dict) and item.get("eligible") is True
|
||||
)
|
||||
|
||||
|
||||
def build_e36_source_catalog_audit(
|
||||
*,
|
||||
profile_path: Path,
|
||||
inventory_path: Path,
|
||||
output_root: Path,
|
||||
) -> E36SourceCatalogAudit:
|
||||
"""Build or reopen one immutable source-catalog admission result."""
|
||||
|
||||
profile_path = profile_path.resolve(strict=True)
|
||||
inventory_path = inventory_path.resolve(strict=True)
|
||||
profile = _read_json(profile_path)
|
||||
inventory = _read_json(inventory_path)
|
||||
_validate_profile(profile)
|
||||
_validate_inventory(inventory)
|
||||
|
||||
baseline = _object(profile.get("baseline"), "E36 baseline")
|
||||
baseline_session_id = _required_string(baseline, "source_session_id")
|
||||
candidates = _array(inventory.get("sources"), "E36 inventory sources")
|
||||
source_ids = {
|
||||
_required_string(_object(item, "E36 source"), "session_id")
|
||||
for item in candidates
|
||||
}
|
||||
if baseline_session_id not in source_ids:
|
||||
raise E36SourceCatalogAuditError("E36 inventory does not contain its baseline")
|
||||
|
||||
identity = {
|
||||
"schema_version": E36_AUDIT_SCHEMA,
|
||||
"profile": profile,
|
||||
"profile_sha256": _sha256(profile_path),
|
||||
"inventory_sha256": _sha256(inventory_path),
|
||||
"producer_sha256": _sha256(Path(__file__)),
|
||||
"authority": _authority(),
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e36-source-catalog-audit-{identity_sha256}"
|
||||
destination = output_root.expanduser().absolute()
|
||||
destination.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
result_root = destination / result_id
|
||||
if result_root.exists():
|
||||
return read_e36_source_catalog_audit(result_root)
|
||||
|
||||
report = _audit_sources(
|
||||
result_id=result_id,
|
||||
profile=profile,
|
||||
inventory=inventory,
|
||||
)
|
||||
staging = destination / f".{result_id}.{os.getpid()}.incomplete"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
_write_json(staging / E36_REPORT_NAME, report)
|
||||
report_artifact = _artifact(staging / E36_REPORT_NAME, "source-catalog-audit-report")
|
||||
manifest = {
|
||||
"schema_version": E36_AUDIT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"status": report["status"],
|
||||
"eligible_source_ids": report["eligible_source_ids"],
|
||||
"artifacts": [report_artifact],
|
||||
"created_at_utc": utc_now_iso(),
|
||||
"authority": _authority(),
|
||||
}
|
||||
_write_json(staging / E36_MANIFEST_NAME, manifest)
|
||||
os.replace(staging, result_root)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return read_e36_source_catalog_audit(result_root)
|
||||
|
||||
|
||||
def read_e36_source_catalog_audit(root: Path) -> E36SourceCatalogAudit:
|
||||
"""Read and fully validate an immutable E36 catalog result."""
|
||||
|
||||
resolved = root.expanduser().resolve(strict=True)
|
||||
if not resolved.is_dir() or _RESULT_ID.fullmatch(resolved.name) is None:
|
||||
raise E36SourceCatalogAuditError("E36 result root is invalid")
|
||||
manifest = _read_json(resolved / E36_MANIFEST_NAME)
|
||||
report = _read_json(resolved / E36_REPORT_NAME)
|
||||
if (
|
||||
manifest.get("schema_version") != E36_AUDIT_SCHEMA
|
||||
or manifest.get("result_id") != resolved.name
|
||||
or report.get("schema_version") != E36_REPORT_SCHEMA
|
||||
or report.get("result_id") != resolved.name
|
||||
or manifest.get("status") != report.get("status")
|
||||
or manifest.get("eligible_source_ids") != report.get("eligible_source_ids")
|
||||
or manifest.get("authority") != _authority()
|
||||
or report.get("authority") != _authority()
|
||||
):
|
||||
raise E36SourceCatalogAuditError("E36 manifest and report are inconsistent")
|
||||
identity = _object(manifest.get("identity"), "E36 identity")
|
||||
expected_identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
if (
|
||||
manifest.get("identity_sha256") != expected_identity_sha256
|
||||
or resolved.name != f"e36-source-catalog-audit-{expected_identity_sha256}"
|
||||
):
|
||||
raise E36SourceCatalogAuditError("E36 result identity is invalid")
|
||||
artifacts = _array(manifest.get("artifacts"), "E36 artifacts")
|
||||
if len(artifacts) != 1:
|
||||
raise E36SourceCatalogAuditError("E36 result has an invalid artifact set")
|
||||
artifact = _object(artifacts[0], "E36 report artifact")
|
||||
if (
|
||||
artifact.get("role") != "source-catalog-audit-report"
|
||||
or artifact.get("path") != E36_REPORT_NAME
|
||||
or artifact.get("sha256") != _sha256(resolved / E36_REPORT_NAME)
|
||||
or artifact.get("byte_length") != (resolved / E36_REPORT_NAME).stat().st_size
|
||||
):
|
||||
raise E36SourceCatalogAuditError("E36 report artifact is invalid")
|
||||
return E36SourceCatalogAudit(
|
||||
result_root=resolved,
|
||||
result_id=resolved.name,
|
||||
manifest=manifest,
|
||||
report=report,
|
||||
)
|
||||
|
||||
|
||||
def _audit_sources(
|
||||
*,
|
||||
result_id: str,
|
||||
profile: dict[str, Any],
|
||||
inventory: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
baseline = _object(profile.get("baseline"), "E36 baseline")
|
||||
baseline_session_id = _required_string(baseline, "source_session_id")
|
||||
eligibility = _object(profile.get("eligibility"), "E36 eligibility")
|
||||
required_modalities = set(
|
||||
_string_array(eligibility.get("required_modalities"), "required modalities")
|
||||
)
|
||||
required_status = _required_string(eligibility, "required_status")
|
||||
minimum_camera_seconds = _number(
|
||||
eligibility.get("minimum_contiguous_camera_seconds"),
|
||||
"minimum camera seconds",
|
||||
)
|
||||
required_origin = _required_string(eligibility, "required_origin")
|
||||
|
||||
audited: list[dict[str, Any]] = []
|
||||
for raw_source in _array(inventory.get("sources"), "E36 inventory sources"):
|
||||
source = _object(raw_source, "E36 source")
|
||||
session_id = _required_string(source, "session_id")
|
||||
if session_id == baseline_session_id:
|
||||
audited.append(
|
||||
{
|
||||
"session_id": session_id,
|
||||
"display_name": _required_string(source, "display_name"),
|
||||
"role": "baseline",
|
||||
"eligible": False,
|
||||
"requirements": {"distinct_from_baseline": False},
|
||||
"blocker_codes": ["baseline-source"],
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
modalities = set(_string_array(source.get("modalities"), "source modalities"))
|
||||
camera = source.get("camera")
|
||||
camera_window_seconds = (
|
||||
0.0
|
||||
if camera is None
|
||||
else _number(
|
||||
_object(camera, "camera evidence").get("longest_contiguous_seconds"),
|
||||
"camera window seconds",
|
||||
)
|
||||
)
|
||||
device = source.get("device_identity")
|
||||
calibration = source.get("calibration_binding")
|
||||
mount = source.get("mount_binding")
|
||||
requirements = {
|
||||
"distinct_from_baseline": True,
|
||||
"accepted_real_source_origin": source.get("origin") == required_origin,
|
||||
"catalog_status_ready": source.get("status") == required_status,
|
||||
"required_modalities_present": required_modalities.issubset(modalities),
|
||||
"contiguous_camera_window_sufficient": (
|
||||
camera_window_seconds >= minimum_camera_seconds
|
||||
),
|
||||
"host_time_identity_present": _valid_sha256(
|
||||
source.get("host_time_identity_sha256")
|
||||
),
|
||||
"device_identity_present": _device_identity_complete(device),
|
||||
"calibration_bound_to_source_session": _binding_complete(
|
||||
calibration,
|
||||
session_id=session_id,
|
||||
identity_key="calibration_identity_sha256",
|
||||
),
|
||||
"mount_bound_to_source_session": _binding_complete(
|
||||
mount,
|
||||
session_id=session_id,
|
||||
identity_key="mount_identity_sha256",
|
||||
),
|
||||
}
|
||||
blocker_codes = [
|
||||
name.replace("_", "-")
|
||||
for name, passed in requirements.items()
|
||||
if not passed
|
||||
]
|
||||
audited.append(
|
||||
{
|
||||
"session_id": session_id,
|
||||
"display_name": _required_string(source, "display_name"),
|
||||
"role": "candidate",
|
||||
"catalog_status": source.get("status"),
|
||||
"modalities": sorted(modalities),
|
||||
"duration_seconds": source.get("duration_seconds"),
|
||||
"longest_contiguous_camera_seconds": camera_window_seconds,
|
||||
"eligible": not blocker_codes,
|
||||
"requirements": requirements,
|
||||
"blocker_codes": blocker_codes,
|
||||
}
|
||||
)
|
||||
|
||||
eligible_ids = [
|
||||
item["session_id"] for item in audited if item.get("eligible") is True
|
||||
]
|
||||
candidates = [item for item in audited if item["role"] == "candidate"]
|
||||
nearest = min(
|
||||
candidates,
|
||||
key=lambda item: (len(item["blocker_codes"]), item["session_id"]),
|
||||
default=None,
|
||||
)
|
||||
status = (
|
||||
"eligible-second-mounted-source-found"
|
||||
if eligible_ids
|
||||
else "blocked-no-eligible-second-mounted-source"
|
||||
)
|
||||
decision: dict[str, Any] = {
|
||||
"e36_replay_authorized": bool(eligible_ids),
|
||||
"eligible_source_count": len(eligible_ids),
|
||||
"retuning_allowed": False,
|
||||
"empty_lab_publication_allowed": False,
|
||||
"new_capture_authorized": False,
|
||||
"public_dataset_substitution_allowed": False,
|
||||
}
|
||||
if nearest is not None:
|
||||
decision["nearest_candidate"] = {
|
||||
"session_id": nearest["session_id"],
|
||||
"display_name": nearest["display_name"],
|
||||
"remaining_blocker_codes": nearest["blocker_codes"],
|
||||
}
|
||||
return {
|
||||
"schema_version": E36_REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"status": status,
|
||||
"catalog_audited_at_utc": inventory["catalog_audited_at_utc"],
|
||||
"source_count": len(audited),
|
||||
"candidate_count": len(candidates),
|
||||
"eligible_source_ids": eligible_ids,
|
||||
"frozen_upstream": baseline["frozen_upstream"],
|
||||
"candidates": audited,
|
||||
"decision": decision,
|
||||
"next_gate": (
|
||||
"run-frozen-e32-e35-transfer"
|
||||
if eligible_ids
|
||||
else "bind-candidate-calibration-and-mount-provenance"
|
||||
),
|
||||
"authority": _authority(),
|
||||
}
|
||||
|
||||
|
||||
def _validate_profile(profile: dict[str, Any]) -> None:
|
||||
if profile.get("schema_version") != E36_PROFILE_SCHEMA:
|
||||
raise E36SourceCatalogAuditError("E36 profile schema is invalid")
|
||||
baseline = _object(profile.get("baseline"), "E36 baseline")
|
||||
_validate_session_id(_required_string(baseline, "source_session_id"))
|
||||
upstream = _object(baseline.get("frozen_upstream"), "E36 frozen upstream")
|
||||
if set(upstream) != {"e31", "e32", "e33", "e34", "e35"}:
|
||||
raise E36SourceCatalogAuditError("E36 frozen upstream set is incomplete")
|
||||
if any(not _required_string(upstream, key) for key in upstream):
|
||||
raise E36SourceCatalogAuditError("E36 frozen upstream identity is invalid")
|
||||
eligibility = _object(profile.get("eligibility"), "E36 eligibility")
|
||||
if (
|
||||
_required_string(eligibility, "required_status") != "ready"
|
||||
or _required_string(eligibility, "required_origin")
|
||||
!= "xgrids-k1.viewer-live.evidence"
|
||||
or set(
|
||||
_string_array(
|
||||
eligibility.get("required_modalities"),
|
||||
"required modalities",
|
||||
)
|
||||
)
|
||||
!= {"point-cloud", "trajectory", "video"}
|
||||
or _number(
|
||||
eligibility.get("minimum_contiguous_camera_seconds"),
|
||||
"minimum camera seconds",
|
||||
)
|
||||
< 60.0
|
||||
or eligibility.get("require_host_time_identity") is not True
|
||||
or eligibility.get("require_device_identity") is not True
|
||||
or eligibility.get("require_session_bound_calibration") is not True
|
||||
or eligibility.get("require_session_bound_mount") is not True
|
||||
or eligibility.get("retuning_allowed") is not False
|
||||
):
|
||||
raise E36SourceCatalogAuditError("E36 eligibility policy is invalid")
|
||||
if profile.get("authority") != _authority():
|
||||
raise E36SourceCatalogAuditError("E36 profile authority is invalid")
|
||||
|
||||
|
||||
def _validate_inventory(inventory: dict[str, Any]) -> None:
|
||||
if inventory.get("schema_version") != E36_INVENTORY_SCHEMA:
|
||||
raise E36SourceCatalogAuditError("E36 inventory schema is invalid")
|
||||
if not _required_string(inventory, "catalog_audited_at_utc").endswith("Z"):
|
||||
raise E36SourceCatalogAuditError("E36 inventory audit time is invalid")
|
||||
sources = _array(inventory.get("sources"), "E36 inventory sources")
|
||||
if not sources:
|
||||
raise E36SourceCatalogAuditError("E36 inventory is empty")
|
||||
seen: set[str] = set()
|
||||
for raw_source in sources:
|
||||
source = _object(raw_source, "E36 source")
|
||||
session_id = _required_string(source, "session_id")
|
||||
_validate_session_id(session_id)
|
||||
if session_id in seen:
|
||||
raise E36SourceCatalogAuditError("E36 inventory has duplicate sessions")
|
||||
seen.add(session_id)
|
||||
_required_string(source, "display_name")
|
||||
_required_string(source, "status")
|
||||
_required_string(source, "origin")
|
||||
_string_array(source.get("modalities"), "source modalities")
|
||||
raw_sha256 = source.get("raw_transport_sha256")
|
||||
if raw_sha256 is not None and not _valid_sha256(raw_sha256):
|
||||
raise E36SourceCatalogAuditError("E36 source transport identity is invalid")
|
||||
|
||||
|
||||
def _binding_complete(
|
||||
value: object,
|
||||
*,
|
||||
session_id: str,
|
||||
identity_key: str,
|
||||
) -> bool:
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
return (
|
||||
value.get("source_session_id") == session_id
|
||||
and _valid_sha256(value.get(identity_key))
|
||||
)
|
||||
|
||||
|
||||
def _device_identity_complete(value: object) -> bool:
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
return _valid_sha256(value.get("device_serial_sha256")) and _valid_sha256(
|
||||
value.get("vendor_device_id_sha256")
|
||||
)
|
||||
|
||||
|
||||
def _authority() -> dict[str, bool]:
|
||||
return {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"transfer_replay_authorized": False,
|
||||
}
|
||||
|
||||
|
||||
def _artifact(path: Path, role: str) -> dict[str, object]:
|
||||
return {
|
||||
"role": role,
|
||||
"path": path.name,
|
||||
"media_type": "application/json",
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise E36SourceCatalogAuditError(f"cannot read E36 JSON: {path.name}") from exc
|
||||
return _object(value, f"E36 JSON {path.name}")
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_text(
|
||||
json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _valid_sha256(value: object) -> bool:
|
||||
return isinstance(value, str) and _SHA256.fullmatch(value) is not None
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise E36SourceCatalogAuditError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _array(value: object, label: str) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
raise E36SourceCatalogAuditError(f"{label} must be an array")
|
||||
return value
|
||||
|
||||
|
||||
def _string_array(value: object, label: str) -> list[str]:
|
||||
values = _array(value, label)
|
||||
if any(not isinstance(item, str) or not item for item in values):
|
||||
raise E36SourceCatalogAuditError(f"{label} must contain strings")
|
||||
return values
|
||||
|
||||
|
||||
def _required_string(value: dict[str, Any], key: str) -> str:
|
||||
item = value.get(key)
|
||||
if not isinstance(item, str) or not item.strip():
|
||||
raise E36SourceCatalogAuditError(f"E36 {key} is invalid")
|
||||
return item
|
||||
|
||||
|
||||
def _number(value: object, label: str) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise E36SourceCatalogAuditError(f"{label} must be numeric")
|
||||
number = float(value)
|
||||
if not math.isfinite(number) or number < 0:
|
||||
raise E36SourceCatalogAuditError(f"{label} must be finite and non-negative")
|
||||
return number
|
||||
|
||||
|
||||
def _validate_session_id(value: str) -> None:
|
||||
if _SESSION_ID.fullmatch(value) is None:
|
||||
raise E36SourceCatalogAuditError("E36 session id is invalid")
|
||||
Reference in New Issue
Block a user