SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(storage): index exact content references

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 15:21:03 +03:00
parent 611853d3af
commit 32c5d0dda5
13 changed files with 1081 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/web/app.py
+7
View File
@@ -669,6 +669,13 @@ app.include_router(
/ "e44"
/ "results"
),
e50_results_root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "e50"
/ "results"
),
)
)
app.include_router(
src/k1link/web/artifact_health_api.py
+96 -8
View File
@@ -18,6 +18,10 @@ from k1link.compute.e44_data_amplification_audit import (
E44DataAmplificationAuditError,
read_e44_data_amplification_audit,
)
from k1link.compute.e50_content_reference_index import (
E50ContentReferenceIndexError,
read_e50_content_reference_index,
)
ARTIFACT_HEALTH_SCHEMA: Final = "missioncore.artifact-health/v1"
DEFAULT_CACHE_SECONDS: Final = 30.0
@@ -77,8 +81,18 @@ class ExactAmplificationAudit(BaseModel):
class ContentReferenceStatus(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
state: Literal["not-indexed"] = "not-indexed"
storage_model: Literal["materialized-copies"] = "materialized-copies"
state: Literal["not-indexed", "indexed"]
storage_model: Literal[
"materialized-copies",
"materialized-copies-with-reference-index",
]
result_id: str | None = None
indexed_at_utc: str | None = None
logical_reference_count: int = Field(default=0, ge=0)
canonical_content_count: int = Field(default=0, ge=0)
duplicate_reference_count: int = Field(default=0, ge=0)
exact_duplicate_bytes: int = Field(default=0, ge=0)
physical_reclamation_applied: Literal[False] = False
storage_migration_authorized: Literal[False] = False
next_gate: str
@@ -102,10 +116,12 @@ class ArtifactHealthService:
runtime_root_provider: RootProvider,
e44_results_root_provider: RootProvider,
*,
e50_results_root_provider: RootProvider | None = None,
cache_seconds: float = DEFAULT_CACHE_SECONDS,
) -> None:
self._runtime_root_provider = runtime_root_provider
self._e44_results_root_provider = e44_results_root_provider
self._e50_results_root_provider = e50_results_root_provider
self._cache_seconds = max(cache_seconds, 0)
self._lock = threading.Lock()
self._cached_at = 0.0
@@ -124,6 +140,10 @@ class ArtifactHealthService:
expected_roots = audit_source[2] if audit_source is not None else {}
inventory, audited_roots = self._scan_runtime(expected_roots, previous)
exact_audit = self._audit_document(audit_source, audited_roots)
content_references = self._content_reference_document(
self._read_latest_reference_index(),
exact_audit,
)
overall_state: Literal["live", "attention", "unavailable"]
if inventory.state == "unavailable":
overall_state = "unavailable"
@@ -140,12 +160,7 @@ class ArtifactHealthService:
overall_state=overall_state,
inventory=inventory,
exact_audit=exact_audit,
content_references=ContentReferenceStatus(
next_gate=(
"Select content-addressed references or chunking only from "
"measured dominant duplicate classes."
)
),
content_references=content_references,
)
self._cached = document
self._cached_at = now
@@ -190,6 +205,37 @@ class ArtifactHealthService:
continue
return None
def _read_latest_reference_index(
self,
) -> tuple[dict[str, Any], dict[str, Any]] | None:
if self._e50_results_root_provider is None:
return None
results_root = self._e50_results_root_provider()
if not results_root.is_dir():
return None
candidates = sorted(
(
candidate
for candidate in results_root.glob("e50-content-reference-index-*")
if candidate.is_dir()
),
key=lambda candidate: candidate.stat().st_mtime_ns,
reverse=True,
)
for candidate in candidates:
try:
index = read_e50_content_reference_index(candidate)
return index.manifest, index.report
except (
E50ContentReferenceIndexError,
OSError,
TypeError,
ValueError,
json.JSONDecodeError,
):
continue
return None
def _scan_runtime(
self,
expected_roots: dict[str, dict[str, int]],
@@ -359,17 +405,59 @@ class ArtifactHealthService:
limitations=tuple(str(item) for item in report.get("limitations", ())),
)
def _content_reference_document(
self,
source: tuple[dict[str, Any], dict[str, Any]] | None,
exact_audit: ExactAmplificationAudit,
) -> ContentReferenceStatus:
if source is None:
return ContentReferenceStatus(
state="not-indexed",
storage_model="materialized-copies",
next_gate=(
"Build an immutable exact-content reference index from the "
"accepted amplification audit."
),
)
manifest, report = source
if report.get("source_e44_result_id") != exact_audit.result_id:
return ContentReferenceStatus(
state="not-indexed",
storage_model="materialized-copies",
next_gate=(
"Rebuild the content reference index from the current exact "
"amplification audit."
),
)
metrics = report["metrics"]
return ContentReferenceStatus(
state="indexed",
storage_model="materialized-copies-with-reference-index",
result_id=str(report["result_id"]),
indexed_at_utc=str(manifest["created_at_utc"]),
logical_reference_count=int(metrics["logical_reference_count"]),
canonical_content_count=int(metrics["canonical_content_count"]),
duplicate_reference_count=int(metrics["duplicate_reference_count"]),
exact_duplicate_bytes=int(metrics["exact_duplicate_bytes"]),
next_gate=(
"Adopt verified reference resolution in future producers before "
"any separately authorized physical reclamation."
),
)
def build_artifact_health_router(
*,
runtime_root_provider: RootProvider,
e44_results_root_provider: RootProvider,
e50_results_root_provider: RootProvider | None = None,
cache_seconds: float = DEFAULT_CACHE_SECONDS,
) -> APIRouter:
router = APIRouter()
service = ArtifactHealthService(
runtime_root_provider,
e44_results_root_provider,
e50_results_root_provider=e50_results_root_provider,
cache_seconds=cache_seconds,
)