feat(storage): index exact content references

2026-07-29 15:21:03 +03:00
parent 611853d3af
commit 32c5d0dda5
13 changed files with 1081 additions and 32 deletions
@@ -6,6 +6,9 @@ from pathlib import Path
 from k1link.compute.e44_data_amplification_audit import (
    build_e44_data_amplification_audit,
 )
+from k1link.compute.e50_content_reference_index import (
+    build_e50_content_reference_index,
+)
 from k1link.web.artifact_health_api import ArtifactHealthService


@@ -23,14 +26,21 @@ def test_artifact_health_reports_live_inventory_and_exact_audit(tmp_path: Path)
    _write(second / "shared.bin", b"shared")
    _write(second / "only-b.bin", b"bb")
    e44_results = runtime / "compute-experiments" / "e44" / "results"
-    build_e44_data_amplification_audit(
+    e44 = build_e44_data_amplification_audit(
        artifact_roots={"e30": first, "e40": second},
        output_root=e44_results,
    )
+    e50_results = runtime / "compute-experiments" / "e50" / "results"
+    e50 = build_e50_content_reference_index(
+        e44_result_root=e44.result_root,
+        artifact_roots={"e30": first, "e40": second},
+        output_root=e50_results,
+    )

    service = ArtifactHealthService(
        lambda: runtime,
        lambda: e44_results,
+        e50_results_root_provider=lambda: e50_results,
        cache_seconds=0,
    )
    document = service.snapshot()
@@ -40,7 +50,13 @@ def test_artifact_health_reports_live_inventory_and_exact_audit(tmp_path: Path)
    assert document.inventory.logical_bytes > 0
    assert document.exact_audit.state == "exact-current"
    assert document.exact_audit.duplicate_bytes == len(b"shared")
-    assert document.content_references.state == "not-indexed"
+    assert document.content_references.state == "indexed"
+    assert document.content_references.result_id == e50.result_id
+    assert document.content_references.logical_reference_count == 4
+    assert document.content_references.canonical_content_count == 3
+    assert document.content_references.duplicate_reference_count == 1
+    assert document.content_references.exact_duplicate_bytes == len(b"shared")
+    assert document.content_references.physical_reclamation_applied is False
    assert document.content_references.storage_migration_authorized is False


@@ -81,3 +97,4 @@ def test_artifact_health_reports_unavailable_runtime(tmp_path: Path) -> None:
    assert document.overall_state == "unavailable"
    assert document.inventory.state == "unavailable"
    assert document.exact_audit.state == "unavailable"
+    assert document.content_references.state == "not-indexed"
@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from k1link.compute.e44_data_amplification_audit import (
+    build_e44_data_amplification_audit,
+)
+from k1link.compute.e50_content_reference_index import (
+    E50ContentReferenceIndexError,
+    build_e50_content_reference_index,
+    read_e50_content_reference_index,
+    resolve_e50_content_reference,
+)
+
+
+def _write(path: Path, payload: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(payload)
+
+
+def test_e50_indexes_exact_references_without_rewriting_sources(
+    tmp_path: Path,
+) -> None:
+    result_root = tmp_path / "e30-result"
+    package_root = tmp_path / "e40-package"
+    shared = b"shared-evidence"
+    _write(result_root / "frames/shared.bin", shared)
+    _write(result_root / "unique.bin", b"result-only")
+    _write(package_root / "input/shared.bin", shared)
+    before = {
+        path: (path.stat().st_size, path.stat().st_mtime_ns, path.read_bytes())
+        for path in (result_root / "frames/shared.bin", package_root / "input/shared.bin")
+    }
+    roots = {"e30": result_root, "e40-package": package_root}
+    e44 = build_e44_data_amplification_audit(
+        artifact_roots=roots,
+        output_root=tmp_path / "e44",
+    )
+
+    result = build_e50_content_reference_index(
+        e44_result_root=e44.result_root,
+        artifact_roots=roots,
+        output_root=tmp_path / "e50",
+    )
+    repeated = read_e50_content_reference_index(result.result_root)
+
+    assert repeated.result_id == result.result_id
+    assert result.report["metrics"]["logical_reference_count"] == 3
+    assert result.report["metrics"]["canonical_content_count"] == 2
+    assert result.report["metrics"]["duplicate_reference_count"] == 1
+    assert result.report["metrics"]["exact_duplicate_bytes"] == len(shared)
+    assert result.report["decision"] == {
+        "content_references_indexed": True,
+        "existing_artifacts_deleted": False,
+        "existing_artifacts_rewritten": False,
+        "physical_reclamation_applied": False,
+        "resolver_requires_explicit_trusted_roots": True,
+        "storage_migration_authorized": False,
+    }
+    assert before == {
+        path: (path.stat().st_size, path.stat().st_mtime_ns, path.read_bytes())
+        for path in before
+    }
+
+
+def test_e50_resolves_duplicate_to_verified_non_package_canonical(
+    tmp_path: Path,
+) -> None:
+    result_root = tmp_path / "e30-result"
+    package_root = tmp_path / "e40-package"
+    _write(result_root / "shared.bin", b"same")
+    _write(package_root / "shared.bin", b"same")
+    roots = {"e30": result_root, "e40-package": package_root}
+    e44 = build_e44_data_amplification_audit(
+        artifact_roots=roots,
+        output_root=tmp_path / "e44",
+    )
+    result = build_e50_content_reference_index(
+        e44_result_root=e44.result_root,
+        artifact_roots=roots,
+        output_root=tmp_path / "e50",
+    )
+
+    canonical = resolve_e50_content_reference(
+        result.result_root,
+        roots,
+        root_label="e40-package",
+        relative_path="shared.bin",
+    )
+    logical = resolve_e50_content_reference(
+        result.result_root,
+        roots,
+        root_label="e40-package",
+        relative_path="shared.bin",
+        prefer_canonical=False,
+    )
+
+    assert canonical == (result_root / "shared.bin").resolve()
+    assert logical == (package_root / "shared.bin").resolve()
+
+
+def test_e50_resolver_fails_closed_after_referenced_content_changes(
+    tmp_path: Path,
+) -> None:
+    first = tmp_path / "first-root"
+    second = tmp_path / "second-root"
+    _write(first / "shared.bin", b"same")
+    _write(second / "shared.bin", b"same")
+    roots = {"first": first, "second": second}
+    e44 = build_e44_data_amplification_audit(
+        artifact_roots=roots,
+        output_root=tmp_path / "e44",
+    )
+    result = build_e50_content_reference_index(
+        e44_result_root=e44.result_root,
+        artifact_roots=roots,
+        output_root=tmp_path / "e50",
+    )
+    _write(second / "shared.bin", b"changed")
+
+    with pytest.raises(E50ContentReferenceIndexError, match="changed"):
+        resolve_e50_content_reference(
+            result.result_root,
+            roots,
+            root_label="second",
+            relative_path="shared.bin",
+            prefer_canonical=False,
+        )
+
+
+def test_e50_rejects_symlinked_source_and_trusted_roots(tmp_path: Path) -> None:
+    first = tmp_path / "first-root"
+    second = tmp_path / "second-root"
+    _write(first / "shared.bin", b"same")
+    _write(second / "shared.bin", b"same")
+    roots = {"first": first, "second": second}
+    e44 = build_e44_data_amplification_audit(
+        artifact_roots=roots,
+        output_root=tmp_path / "e44",
+    )
+    first_link = tmp_path / "first-link"
+    first_link.symlink_to(first, target_is_directory=True)
+
+    with pytest.raises(E50ContentReferenceIndexError, match="symlink"):
+        build_e50_content_reference_index(
+            e44_result_root=e44.result_root,
+            artifact_roots={"first": first_link, "second": second},
+            output_root=tmp_path / "rejected-e50",
+        )
+
+    result = build_e50_content_reference_index(
+        e44_result_root=e44.result_root,
+        artifact_roots=roots,
+        output_root=tmp_path / "e50",
+    )
+    with pytest.raises(E50ContentReferenceIndexError, match="symlink"):
+        resolve_e50_content_reference(
+            result.result_root,
+            {"first": first_link, "second": second},
+            root_label="first",
+            relative_path="shared.bin",
+        )