feat(storage): index exact content references
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,571 @@
|
||||
"""Non-destructive exact-content references over an accepted E44 catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.compute.e44_data_amplification_audit import (
|
||||
analyze_data_amplification,
|
||||
read_e44_data_amplification_audit,
|
||||
)
|
||||
|
||||
E50_RESULT_SCHEMA: Final = "missioncore.e50-content-reference-index/v1"
|
||||
E50_REPORT_SCHEMA: Final = "missioncore.e50-content-reference-report/v1"
|
||||
CONTENT_REFERENCE_SCHEMA: Final = "missioncore.content-reference/v1"
|
||||
E50_INDEX_NAME: Final = "content-references.jsonl"
|
||||
E50_REPORT_NAME: Final = "content-reference-report.json"
|
||||
E50_MANIFEST_NAME: Final = "manifest.json"
|
||||
CANONICAL_POLICY_ID: Final = "prefer-non-package-then-lexical/v1"
|
||||
|
||||
_LABEL = re.compile(r"^[a-z0-9][a-z0-9._-]{1,63}$")
|
||||
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class E50ContentReferenceIndexError(RuntimeError):
|
||||
"""An E50 source catalog, reference, or immutable result is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class E50ContentReferenceIndex:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
manifest: dict[str, Any]
|
||||
report: dict[str, Any]
|
||||
references: tuple[dict[str, Any], ...]
|
||||
|
||||
|
||||
def build_e50_content_reference_index(
|
||||
*,
|
||||
e44_result_root: Path,
|
||||
artifact_roots: dict[str, Path],
|
||||
output_root: Path,
|
||||
) -> E50ContentReferenceIndex:
|
||||
"""Index exact content without copying, linking, rewriting, or deleting sources."""
|
||||
|
||||
e44 = read_e44_data_amplification_audit(e44_result_root)
|
||||
expected_rows = e44.manifest["identity"]["artifact_roots"]
|
||||
if not isinstance(expected_rows, list):
|
||||
raise E50ContentReferenceIndexError("E50 E44 root identity is invalid")
|
||||
expected_roots = {
|
||||
str(row["label"]): {
|
||||
"root_name": str(row["root_name"]),
|
||||
"catalog_sha256": str(row["catalog_sha256"]),
|
||||
}
|
||||
for row in expected_rows
|
||||
if isinstance(row, dict)
|
||||
}
|
||||
if (
|
||||
len(expected_roots) != len(expected_rows)
|
||||
or set(artifact_roots) != set(expected_roots)
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 artifact root coverage changed")
|
||||
|
||||
resolved_output = output_root.expanduser().resolve(strict=False)
|
||||
resolved_roots: dict[str, Path] = {}
|
||||
for label, source in artifact_roots.items():
|
||||
if _LABEL.fullmatch(label) is None:
|
||||
raise E50ContentReferenceIndexError("E50 artifact label is invalid")
|
||||
source_path = source.expanduser().absolute()
|
||||
if source_path.is_symlink():
|
||||
raise E50ContentReferenceIndexError("E50 artifact root is a symlink")
|
||||
root = source_path.resolve(strict=True)
|
||||
expected = expected_roots[label]
|
||||
if (
|
||||
not root.is_dir()
|
||||
or root.name != expected["root_name"]
|
||||
or resolved_output == root
|
||||
or resolved_output.is_relative_to(root)
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 artifact root is invalid")
|
||||
resolved_roots[label] = root
|
||||
|
||||
files = _catalog_files(resolved_roots)
|
||||
for label in sorted(resolved_roots):
|
||||
rows = [row for row in files if row["root"] == label]
|
||||
observed = hashlib.sha256(_canonical_json(rows)).hexdigest()
|
||||
if observed != expected_roots[label]["catalog_sha256"]:
|
||||
raise E50ContentReferenceIndexError(
|
||||
f"E50 source catalog changed for {label}"
|
||||
)
|
||||
analysis = analyze_data_amplification(files)
|
||||
analysis_sha256 = hashlib.sha256(_canonical_json(analysis)).hexdigest()
|
||||
if analysis_sha256 != e44.manifest["identity"]["analysis_sha256"]:
|
||||
raise E50ContentReferenceIndexError("E50 no longer reproduces the E44 audit")
|
||||
|
||||
references = _build_references(files, resolved_roots)
|
||||
index_payload = b"".join(_canonical_json(row) + b"\n" for row in references)
|
||||
index_sha256 = hashlib.sha256(index_payload).hexdigest()
|
||||
root_identity = [
|
||||
{
|
||||
"label": label,
|
||||
"root_name": expected_roots[label]["root_name"],
|
||||
"catalog_sha256": expected_roots[label]["catalog_sha256"],
|
||||
}
|
||||
for label in sorted(expected_roots)
|
||||
]
|
||||
identity = {
|
||||
"schema_version": E50_RESULT_SCHEMA,
|
||||
"source_e44_result_id": e44.result_id,
|
||||
"source_e44_identity_sha256": e44.manifest["identity_sha256"],
|
||||
"artifact_roots": root_identity,
|
||||
"canonical_policy_id": CANONICAL_POLICY_ID,
|
||||
"reference_schema": CONTENT_REFERENCE_SCHEMA,
|
||||
"reference_count": len(references),
|
||||
"reference_index_sha256": index_sha256,
|
||||
"producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e50-content-reference-index-{identity_sha256}"
|
||||
destination = resolved_output / result_id
|
||||
if destination.exists():
|
||||
return read_e50_content_reference_index(destination)
|
||||
|
||||
unique_content = {str(row["content_id"]) for row in references}
|
||||
canonical_references = sum(
|
||||
row["relation"] == "canonical" for row in references
|
||||
)
|
||||
report = {
|
||||
"schema_version": E50_REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"status": "completed-non-destructive-content-reference-index",
|
||||
"source_e44_result_id": e44.result_id,
|
||||
"metrics": {
|
||||
"root_count": len(resolved_roots),
|
||||
"logical_reference_count": len(references),
|
||||
"canonical_content_count": len(unique_content),
|
||||
"canonical_reference_count": canonical_references,
|
||||
"duplicate_reference_count": len(references) - len(unique_content),
|
||||
"duplicate_content_groups": analysis["duplicate_content_groups"],
|
||||
"logical_bytes": analysis["logical_bytes"],
|
||||
"addressable_unique_bytes": analysis["unique_content_bytes"],
|
||||
"exact_duplicate_bytes": analysis["duplicate_bytes"],
|
||||
"amplification_ratio": analysis["amplification_ratio"],
|
||||
},
|
||||
"decision": {
|
||||
"content_references_indexed": True,
|
||||
"existing_artifacts_rewritten": False,
|
||||
"existing_artifacts_deleted": False,
|
||||
"physical_reclamation_applied": False,
|
||||
"storage_migration_authorized": False,
|
||||
"resolver_requires_explicit_trusted_roots": True,
|
||||
},
|
||||
"limitations": [
|
||||
"the index removes duplicate namespace semantics, not existing physical files",
|
||||
"physical reclamation requires a separately authorized producer migration",
|
||||
"references are valid only while exact root name, path, size and SHA-256 match",
|
||||
],
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
_write_bytes(staging / E50_INDEX_NAME, index_payload)
|
||||
_write_json(staging / E50_REPORT_NAME, report)
|
||||
manifest = {
|
||||
"schema_version": E50_RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": _utc_now(),
|
||||
"acceptance_state": "accepted-reference-index-only",
|
||||
"artifacts": [
|
||||
_artifact(staging / E50_INDEX_NAME, "content-reference-index"),
|
||||
_artifact(staging / E50_REPORT_NAME, "content-reference-report"),
|
||||
],
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
_write_json(staging / E50_MANIFEST_NAME, manifest)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return read_e50_content_reference_index(destination)
|
||||
|
||||
|
||||
def read_e50_content_reference_index(root: Path) -> E50ContentReferenceIndex:
|
||||
"""Read and fully validate one immutable E50 result."""
|
||||
|
||||
source = root.expanduser().absolute()
|
||||
if source.is_symlink():
|
||||
raise E50ContentReferenceIndexError("E50 result root is a symlink")
|
||||
resolved = source.resolve(strict=True)
|
||||
manifest = _read_json(resolved / E50_MANIFEST_NAME)
|
||||
identity = _object(manifest.get("identity"), "E50 identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
manifest.get("schema_version") != E50_RESULT_SCHEMA
|
||||
or not isinstance(identity_sha256, str)
|
||||
or _SHA256.fullmatch(identity_sha256) is None
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("result_id")
|
||||
!= f"e50-content-reference-index-{identity_sha256}"
|
||||
or resolved.name != manifest.get("result_id")
|
||||
or manifest.get("acceptance_state") != "accepted-reference-index-only"
|
||||
or manifest.get("authority") != _AUTHORITY
|
||||
or identity.get("canonical_policy_id") != CANONICAL_POLICY_ID
|
||||
or identity.get("reference_schema") != CONTENT_REFERENCE_SCHEMA
|
||||
or identity.get("authority") != _AUTHORITY
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 result identity is invalid")
|
||||
artifacts = manifest.get("artifacts")
|
||||
expected_artifacts = {
|
||||
E50_INDEX_NAME: "content-reference-index",
|
||||
E50_REPORT_NAME: "content-reference-report",
|
||||
}
|
||||
if not isinstance(artifacts, list) or len(artifacts) != 2:
|
||||
raise E50ContentReferenceIndexError("E50 artifact set is invalid")
|
||||
observed_artifacts: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E50ContentReferenceIndexError("E50 artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_artifacts
|
||||
or relative in observed_artifacts
|
||||
or row.get("role") != expected_artifacts[relative]
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 artifact content changed")
|
||||
observed_artifacts.add(relative)
|
||||
if observed_artifacts != set(expected_artifacts):
|
||||
raise E50ContentReferenceIndexError("E50 artifact coverage changed")
|
||||
|
||||
index_payload = (resolved / E50_INDEX_NAME).read_bytes()
|
||||
if hashlib.sha256(index_payload).hexdigest() != identity.get(
|
||||
"reference_index_sha256"
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 reference index digest changed")
|
||||
references = tuple(
|
||||
_read_reference_line(line)
|
||||
for line in index_payload.splitlines()
|
||||
if line
|
||||
)
|
||||
_validate_reference_set(references)
|
||||
if identity.get("reference_count") != len(references):
|
||||
raise E50ContentReferenceIndexError("E50 reference count changed")
|
||||
|
||||
report = _read_json(resolved / E50_REPORT_NAME)
|
||||
metrics = _object(report.get("metrics"), "E50 metrics")
|
||||
unique_content = {str(row["content_id"]) for row in references}
|
||||
if (
|
||||
report.get("schema_version") != E50_REPORT_SCHEMA
|
||||
or report.get("result_id") != resolved.name
|
||||
or report.get("identity_sha256") != identity_sha256
|
||||
or report.get("source_e44_result_id")
|
||||
!= identity.get("source_e44_result_id")
|
||||
or metrics.get("logical_reference_count") != len(references)
|
||||
or metrics.get("canonical_content_count") != len(unique_content)
|
||||
or metrics.get("canonical_reference_count") != len(unique_content)
|
||||
or report.get("decision", {}).get("content_references_indexed") is not True
|
||||
or report.get("decision", {}).get("existing_artifacts_rewritten") is not False
|
||||
or report.get("decision", {}).get("existing_artifacts_deleted") is not False
|
||||
or report.get("decision", {}).get("physical_reclamation_applied") is not False
|
||||
or report.get("decision", {}).get("storage_migration_authorized") is not False
|
||||
or report.get("authority") != _AUTHORITY
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 report is invalid")
|
||||
return E50ContentReferenceIndex(
|
||||
result_id=resolved.name,
|
||||
result_root=resolved,
|
||||
manifest=manifest,
|
||||
report=report,
|
||||
references=references,
|
||||
)
|
||||
|
||||
|
||||
def resolve_e50_content_reference(
|
||||
index_root: Path,
|
||||
artifact_roots: dict[str, Path],
|
||||
*,
|
||||
root_label: str,
|
||||
relative_path: str,
|
||||
prefer_canonical: bool = True,
|
||||
) -> Path:
|
||||
"""Resolve one indexed logical path through explicit trusted roots."""
|
||||
|
||||
index = read_e50_content_reference_index(index_root)
|
||||
selected = next(
|
||||
(
|
||||
row
|
||||
for row in index.references
|
||||
if row["logical"]["root_label"] == root_label
|
||||
and row["logical"]["path"] == relative_path
|
||||
),
|
||||
None,
|
||||
)
|
||||
if selected is None:
|
||||
raise E50ContentReferenceIndexError("E50 logical reference was not indexed")
|
||||
locator = selected["canonical"] if prefer_canonical else selected["logical"]
|
||||
label = str(locator["root_label"])
|
||||
root = artifact_roots.get(label)
|
||||
if root is None:
|
||||
raise E50ContentReferenceIndexError("E50 trusted root is missing")
|
||||
root_path = root.expanduser().absolute()
|
||||
if root_path.is_symlink():
|
||||
raise E50ContentReferenceIndexError("E50 trusted root is a symlink")
|
||||
resolved_root = root_path.resolve(strict=True)
|
||||
if (
|
||||
not resolved_root.is_dir()
|
||||
or resolved_root.name != locator["root_name"]
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 trusted root identity changed")
|
||||
path = _regular_member(resolved_root, str(locator["path"]))
|
||||
if (
|
||||
path.stat().st_size != selected["byte_length"]
|
||||
or _sha256(path) != selected["sha256"]
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 referenced content changed")
|
||||
return path
|
||||
|
||||
|
||||
def _catalog_files(roots: dict[str, Path]) -> list[dict[str, Any]]:
|
||||
files: list[dict[str, Any]] = []
|
||||
for label, root in sorted(roots.items()):
|
||||
for path in sorted(root.rglob("*")):
|
||||
if path.is_symlink():
|
||||
raise E50ContentReferenceIndexError("E50 input contains a symlink")
|
||||
if not path.is_file():
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"root": label,
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
"kind": _artifact_kind(path),
|
||||
}
|
||||
)
|
||||
if not files:
|
||||
raise E50ContentReferenceIndexError("E50 artifact roots are empty")
|
||||
return files
|
||||
|
||||
|
||||
def _build_references(
|
||||
files: list[dict[str, Any]],
|
||||
roots: dict[str, Path],
|
||||
) -> tuple[dict[str, Any], ...]:
|
||||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for row in files:
|
||||
grouped[str(row["sha256"])].append(row)
|
||||
canonical_by_digest = {
|
||||
digest: min(rows, key=_canonical_sort_key)
|
||||
for digest, rows in grouped.items()
|
||||
}
|
||||
references = []
|
||||
for row in sorted(files, key=lambda item: (item["root"], item["path"])):
|
||||
digest = str(row["sha256"])
|
||||
canonical = canonical_by_digest[digest]
|
||||
logical = _locator(row, roots)
|
||||
canonical_locator = _locator(canonical, roots)
|
||||
references.append(
|
||||
{
|
||||
"schema_version": CONTENT_REFERENCE_SCHEMA,
|
||||
"content_id": f"sha256:{digest}",
|
||||
"algorithm": "sha256",
|
||||
"sha256": digest,
|
||||
"byte_length": int(row["byte_length"]),
|
||||
"kind": str(row["kind"]),
|
||||
"logical": logical,
|
||||
"canonical": canonical_locator,
|
||||
"relation": (
|
||||
"canonical" if logical == canonical_locator else "exact-reference"
|
||||
),
|
||||
}
|
||||
)
|
||||
return tuple(references)
|
||||
|
||||
|
||||
def _canonical_sort_key(row: dict[str, Any]) -> tuple[int, str, str]:
|
||||
label = str(row["root"])
|
||||
package_penalty = int(label.endswith("-package"))
|
||||
return package_penalty, label, str(row["path"])
|
||||
|
||||
|
||||
def _locator(row: dict[str, Any], roots: dict[str, Path]) -> dict[str, str]:
|
||||
label = str(row["root"])
|
||||
return {
|
||||
"root_label": label,
|
||||
"root_name": roots[label].name,
|
||||
"path": str(row["path"]),
|
||||
}
|
||||
|
||||
|
||||
def _read_reference_line(line: bytes) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise E50ContentReferenceIndexError("E50 reference row is invalid") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise E50ContentReferenceIndexError("E50 reference row must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_reference_set(references: tuple[dict[str, Any], ...]) -> None:
|
||||
logical_keys: set[tuple[str, str]] = set()
|
||||
canonical_by_content: dict[str, dict[str, str]] = {}
|
||||
canonical_counts: defaultdict[str, int] = defaultdict(int)
|
||||
for row in references:
|
||||
logical = _object(row.get("logical"), "E50 logical locator")
|
||||
canonical = _object(row.get("canonical"), "E50 canonical locator")
|
||||
digest = row.get("sha256")
|
||||
content_id = row.get("content_id")
|
||||
byte_length = row.get("byte_length")
|
||||
relation = row.get("relation")
|
||||
if (
|
||||
row.get("schema_version") != CONTENT_REFERENCE_SCHEMA
|
||||
or row.get("algorithm") != "sha256"
|
||||
or not isinstance(digest, str)
|
||||
or _SHA256.fullmatch(digest) is None
|
||||
or content_id != f"sha256:{digest}"
|
||||
or not isinstance(byte_length, int)
|
||||
or isinstance(byte_length, bool)
|
||||
or byte_length < 0
|
||||
or not isinstance(row.get("kind"), str)
|
||||
or relation not in {"canonical", "exact-reference"}
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 content reference is invalid")
|
||||
for locator in (logical, canonical):
|
||||
label = locator.get("root_label")
|
||||
root_name = locator.get("root_name")
|
||||
relative = locator.get("path")
|
||||
if (
|
||||
not isinstance(label, str)
|
||||
or _LABEL.fullmatch(label) is None
|
||||
or not isinstance(root_name, str)
|
||||
or not root_name
|
||||
or not isinstance(relative, str)
|
||||
or not relative
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
):
|
||||
raise E50ContentReferenceIndexError("E50 locator is invalid")
|
||||
logical_key = (str(logical["root_label"]), str(logical["path"]))
|
||||
if logical_key in logical_keys:
|
||||
raise E50ContentReferenceIndexError("E50 logical reference is duplicated")
|
||||
logical_keys.add(logical_key)
|
||||
previous = canonical_by_content.setdefault(str(content_id), canonical)
|
||||
if previous != canonical:
|
||||
raise E50ContentReferenceIndexError("E50 canonical reference changed")
|
||||
if relation == "canonical":
|
||||
if logical != canonical:
|
||||
raise E50ContentReferenceIndexError("E50 canonical relation is invalid")
|
||||
canonical_counts[str(content_id)] += 1
|
||||
elif logical == canonical:
|
||||
raise E50ContentReferenceIndexError("E50 exact reference is canonical")
|
||||
if not references or any(count != 1 for count in canonical_counts.values()):
|
||||
raise E50ContentReferenceIndexError("E50 canonical coverage is invalid")
|
||||
if set(canonical_counts) != set(canonical_by_content):
|
||||
raise E50ContentReferenceIndexError("E50 content coverage is invalid")
|
||||
|
||||
|
||||
def _regular_member(root: Path, relative: str) -> Path:
|
||||
path = Path(relative)
|
||||
if path.is_absolute() or not path.parts or ".." in path.parts:
|
||||
raise E50ContentReferenceIndexError("E50 reference path is invalid")
|
||||
candidate = root
|
||||
for part in path.parts:
|
||||
candidate = candidate / part
|
||||
if candidate.is_symlink():
|
||||
raise E50ContentReferenceIndexError("E50 reference crosses a symlink")
|
||||
resolved = candidate.resolve(strict=True)
|
||||
if not resolved.is_file() or not resolved.is_relative_to(root):
|
||||
raise E50ContentReferenceIndexError("E50 reference is not a regular member")
|
||||
return resolved
|
||||
|
||||
|
||||
def _artifact_kind(path: Path) -> str:
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in {".jpg", ".jpeg", ".png", ".webp"}:
|
||||
return "camera-image"
|
||||
if suffix in {".npy", ".npz", ".las", ".laz", ".pcd", ".ply"}:
|
||||
return "point-or-array"
|
||||
if suffix in {".mp4", ".mkv", ".mov"}:
|
||||
return "video"
|
||||
if suffix == ".rrd":
|
||||
return "rerun"
|
||||
if suffix in {".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"}:
|
||||
return "metadata-or-report"
|
||||
if suffix in {".py", ".ps1", ".sh"}:
|
||||
return "runtime-source"
|
||||
return "other"
|
||||
|
||||
|
||||
def _artifact(path: Path, role: str) -> dict[str, Any]:
|
||||
return {
|
||||
"path": path.name,
|
||||
"role": role,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise E50ContentReferenceIndexError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E50ContentReferenceIndexError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(value, stream, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def _write_bytes(path: Path, value: bytes) -> None:
|
||||
with path.open("xb") as stream:
|
||||
stream.write(value)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
@@ -669,6 +669,13 @@ app.include_router(
|
||||
/ "e44"
|
||||
/ "results"
|
||||
),
|
||||
e50_results_root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e50"
|
||||
/ "results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
|
||||
@@ -18,6 +18,10 @@ from k1link.compute.e44_data_amplification_audit import (
|
||||
E44DataAmplificationAuditError,
|
||||
read_e44_data_amplification_audit,
|
||||
)
|
||||
from k1link.compute.e50_content_reference_index import (
|
||||
E50ContentReferenceIndexError,
|
||||
read_e50_content_reference_index,
|
||||
)
|
||||
|
||||
ARTIFACT_HEALTH_SCHEMA: Final = "missioncore.artifact-health/v1"
|
||||
DEFAULT_CACHE_SECONDS: Final = 30.0
|
||||
@@ -77,8 +81,18 @@ class ExactAmplificationAudit(BaseModel):
|
||||
class ContentReferenceStatus(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
state: Literal["not-indexed"] = "not-indexed"
|
||||
storage_model: Literal["materialized-copies"] = "materialized-copies"
|
||||
state: Literal["not-indexed", "indexed"]
|
||||
storage_model: Literal[
|
||||
"materialized-copies",
|
||||
"materialized-copies-with-reference-index",
|
||||
]
|
||||
result_id: str | None = None
|
||||
indexed_at_utc: str | None = None
|
||||
logical_reference_count: int = Field(default=0, ge=0)
|
||||
canonical_content_count: int = Field(default=0, ge=0)
|
||||
duplicate_reference_count: int = Field(default=0, ge=0)
|
||||
exact_duplicate_bytes: int = Field(default=0, ge=0)
|
||||
physical_reclamation_applied: Literal[False] = False
|
||||
storage_migration_authorized: Literal[False] = False
|
||||
next_gate: str
|
||||
|
||||
@@ -102,10 +116,12 @@ class ArtifactHealthService:
|
||||
runtime_root_provider: RootProvider,
|
||||
e44_results_root_provider: RootProvider,
|
||||
*,
|
||||
e50_results_root_provider: RootProvider | None = None,
|
||||
cache_seconds: float = DEFAULT_CACHE_SECONDS,
|
||||
) -> None:
|
||||
self._runtime_root_provider = runtime_root_provider
|
||||
self._e44_results_root_provider = e44_results_root_provider
|
||||
self._e50_results_root_provider = e50_results_root_provider
|
||||
self._cache_seconds = max(cache_seconds, 0)
|
||||
self._lock = threading.Lock()
|
||||
self._cached_at = 0.0
|
||||
@@ -124,6 +140,10 @@ class ArtifactHealthService:
|
||||
expected_roots = audit_source[2] if audit_source is not None else {}
|
||||
inventory, audited_roots = self._scan_runtime(expected_roots, previous)
|
||||
exact_audit = self._audit_document(audit_source, audited_roots)
|
||||
content_references = self._content_reference_document(
|
||||
self._read_latest_reference_index(),
|
||||
exact_audit,
|
||||
)
|
||||
overall_state: Literal["live", "attention", "unavailable"]
|
||||
if inventory.state == "unavailable":
|
||||
overall_state = "unavailable"
|
||||
@@ -140,12 +160,7 @@ class ArtifactHealthService:
|
||||
overall_state=overall_state,
|
||||
inventory=inventory,
|
||||
exact_audit=exact_audit,
|
||||
content_references=ContentReferenceStatus(
|
||||
next_gate=(
|
||||
"Select content-addressed references or chunking only from "
|
||||
"measured dominant duplicate classes."
|
||||
)
|
||||
),
|
||||
content_references=content_references,
|
||||
)
|
||||
self._cached = document
|
||||
self._cached_at = now
|
||||
@@ -190,6 +205,37 @@ class ArtifactHealthService:
|
||||
continue
|
||||
return None
|
||||
|
||||
def _read_latest_reference_index(
|
||||
self,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
if self._e50_results_root_provider is None:
|
||||
return None
|
||||
results_root = self._e50_results_root_provider()
|
||||
if not results_root.is_dir():
|
||||
return None
|
||||
candidates = sorted(
|
||||
(
|
||||
candidate
|
||||
for candidate in results_root.glob("e50-content-reference-index-*")
|
||||
if candidate.is_dir()
|
||||
),
|
||||
key=lambda candidate: candidate.stat().st_mtime_ns,
|
||||
reverse=True,
|
||||
)
|
||||
for candidate in candidates:
|
||||
try:
|
||||
index = read_e50_content_reference_index(candidate)
|
||||
return index.manifest, index.report
|
||||
except (
|
||||
E50ContentReferenceIndexError,
|
||||
OSError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
continue
|
||||
return None
|
||||
|
||||
def _scan_runtime(
|
||||
self,
|
||||
expected_roots: dict[str, dict[str, int]],
|
||||
@@ -359,17 +405,59 @@ class ArtifactHealthService:
|
||||
limitations=tuple(str(item) for item in report.get("limitations", ())),
|
||||
)
|
||||
|
||||
def _content_reference_document(
|
||||
self,
|
||||
source: tuple[dict[str, Any], dict[str, Any]] | None,
|
||||
exact_audit: ExactAmplificationAudit,
|
||||
) -> ContentReferenceStatus:
|
||||
if source is None:
|
||||
return ContentReferenceStatus(
|
||||
state="not-indexed",
|
||||
storage_model="materialized-copies",
|
||||
next_gate=(
|
||||
"Build an immutable exact-content reference index from the "
|
||||
"accepted amplification audit."
|
||||
),
|
||||
)
|
||||
manifest, report = source
|
||||
if report.get("source_e44_result_id") != exact_audit.result_id:
|
||||
return ContentReferenceStatus(
|
||||
state="not-indexed",
|
||||
storage_model="materialized-copies",
|
||||
next_gate=(
|
||||
"Rebuild the content reference index from the current exact "
|
||||
"amplification audit."
|
||||
),
|
||||
)
|
||||
metrics = report["metrics"]
|
||||
return ContentReferenceStatus(
|
||||
state="indexed",
|
||||
storage_model="materialized-copies-with-reference-index",
|
||||
result_id=str(report["result_id"]),
|
||||
indexed_at_utc=str(manifest["created_at_utc"]),
|
||||
logical_reference_count=int(metrics["logical_reference_count"]),
|
||||
canonical_content_count=int(metrics["canonical_content_count"]),
|
||||
duplicate_reference_count=int(metrics["duplicate_reference_count"]),
|
||||
exact_duplicate_bytes=int(metrics["exact_duplicate_bytes"]),
|
||||
next_gate=(
|
||||
"Adopt verified reference resolution in future producers before "
|
||||
"any separately authorized physical reclamation."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_artifact_health_router(
|
||||
*,
|
||||
runtime_root_provider: RootProvider,
|
||||
e44_results_root_provider: RootProvider,
|
||||
e50_results_root_provider: RootProvider | None = None,
|
||||
cache_seconds: float = DEFAULT_CACHE_SECONDS,
|
||||
) -> APIRouter:
|
||||
router = APIRouter()
|
||||
service = ArtifactHealthService(
|
||||
runtime_root_provider,
|
||||
e44_results_root_provider,
|
||||
e50_results_root_provider=e50_results_root_provider,
|
||||
cache_seconds=cache_seconds,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user