feat(lab): publish M4.8T quality evidence
This commit is contained in:
DCCONSTRUCTIONS
@@ -322,6 +322,7 @@ def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]:
|
||||
"canonical.e46j-raw-fisheye-realtime/v1": _run_e46j,
|
||||
"experimental.e47-semantic-slam-shadow/v1": _run_e47,
|
||||
"experimental.m48s-fixed-class-detector/v1": _run_m48s_fixed_class_detector,
|
||||
"experimental.m48t-risk-quality-temporal/v1": _run_m48t_risk_quality_temporal,
|
||||
}
|
||||
|
||||
|
||||
@@ -342,6 +343,21 @@ def _run_m48s_fixed_class_detector(
|
||||
)
|
||||
|
||||
|
||||
def _run_m48t_risk_quality_temporal(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
from k1link.laboratory.m48t_risk_quality_lab import build_m48t_risk_quality_lab
|
||||
|
||||
result = build_m48t_risk_quality_lab(
|
||||
repository_root=request.inputs["repository_root"],
|
||||
output_root=request.output_root,
|
||||
)
|
||||
return LaboratoryAdapterResult(
|
||||
result_root=result.result_root,
|
||||
result_id=result.result_id,
|
||||
)
|
||||
|
||||
|
||||
def _run_m48_small_static_passage_regression(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Seal the M4.8T semantic-quality and temporal-identity evidence as a LAB result."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.perception.fixed_class_detector_tournament import (
|
||||
canonical_json,
|
||||
false_authority,
|
||||
sha256_path,
|
||||
)
|
||||
|
||||
LAB_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-lab/v1"
|
||||
REPORT_SCHEMA: Final = "missioncore.m48t-risk-quality-temporal-report/v1"
|
||||
CATALOG_SCHEMA: Final = "missioncore.m48t-risk-quality-review-catalog/v1"
|
||||
RESULT_PREFIX: Final = "m48t-risk-quality-temporal-lab-"
|
||||
PROFILE_RELATIVE_PATH: Final = Path(
|
||||
"config/perception/m48t-risk-quality-temporal-v1.json"
|
||||
)
|
||||
WORKER_RELATIVE_ROOT: Final = Path(
|
||||
".runtime/worker-results/m48t-risk-quality-coco2017-val-full-v1"
|
||||
)
|
||||
TEMPORAL_LEDGER_RELATIVE_PATH: Final = Path(
|
||||
".runtime/m48s-reference-graph-shadow/full-replay-d85983f1/frames.jsonl"
|
||||
)
|
||||
|
||||
|
||||
class M48TRiskQualityLabError(RuntimeError):
|
||||
"""Raised when the M4.8T evidence cannot be sealed honestly."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class M48TRiskQualityLabResult:
|
||||
result_root: Path
|
||||
result_id: str
|
||||
manifest: dict[str, Any]
|
||||
|
||||
|
||||
def build_m48t_risk_quality_lab(
|
||||
*,
|
||||
repository_root: Path,
|
||||
output_root: Path,
|
||||
) -> M48TRiskQualityLabResult:
|
||||
repository = repository_root.expanduser().resolve(strict=True)
|
||||
profile_path = repository / PROFILE_RELATIVE_PATH
|
||||
worker_root = repository / WORKER_RELATIVE_ROOT
|
||||
temporal_ledger_path = repository / TEMPORAL_LEDGER_RELATIVE_PATH
|
||||
quality_path = worker_root / "result.json"
|
||||
temporal_path = worker_root / "temporal-semantic-shadow.json"
|
||||
predictions_path = worker_root / "predictions.jsonl"
|
||||
failures_path = worker_root / "failures.jsonl"
|
||||
review_root = worker_root / "review"
|
||||
for path in (
|
||||
profile_path,
|
||||
quality_path,
|
||||
temporal_path,
|
||||
predictions_path,
|
||||
failures_path,
|
||||
temporal_ledger_path,
|
||||
):
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise M48TRiskQualityLabError(f"required M4.8T evidence is missing: {path.name}")
|
||||
if review_root.is_symlink() or not review_root.is_dir():
|
||||
raise M48TRiskQualityLabError("M4.8T review evidence is missing")
|
||||
|
||||
profile = _read_object(profile_path)
|
||||
quality = _read_object(quality_path)
|
||||
temporal = _read_object(temporal_path)
|
||||
review_paths = sorted(review_root.glob("review-*.jpg"))
|
||||
_validate_inputs(
|
||||
profile=profile,
|
||||
profile_path=profile_path,
|
||||
quality=quality,
|
||||
predictions_path=predictions_path,
|
||||
failures_path=failures_path,
|
||||
temporal=temporal,
|
||||
temporal_ledger_path=temporal_ledger_path,
|
||||
review_paths=review_paths,
|
||||
)
|
||||
|
||||
method = _method(profile, quality)
|
||||
identity = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"profile": {
|
||||
"profile_id": profile["profile_id"],
|
||||
"sha256": sha256_path(profile_path),
|
||||
},
|
||||
"source": {
|
||||
"quality_dataset_id": quality["dataset"]["dataset_id"],
|
||||
"quality_report_identity_sha256": quality["report_identity_sha256"],
|
||||
"quality_report_sha256": sha256_path(quality_path),
|
||||
"predictions_sha256": sha256_path(predictions_path),
|
||||
"failures_sha256": sha256_path(failures_path),
|
||||
"temporal_source_id": temporal["source"]["source_id"],
|
||||
"temporal_ledger_sha256": sha256_path(temporal_ledger_path),
|
||||
"temporal_shadow_sha256": sha256_path(temporal_path),
|
||||
},
|
||||
"method": method,
|
||||
"authority": false_authority(),
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = RESULT_PREFIX + identity_sha256
|
||||
completed_ns = quality["execution"]["completed_at_unix_ns"]
|
||||
created_at_utc = (
|
||||
datetime.fromtimestamp(completed_ns / 1_000_000_000, UTC)
|
||||
.isoformat(timespec="microseconds")
|
||||
.replace("+00:00", "Z")
|
||||
)
|
||||
|
||||
root = output_root.expanduser().absolute()
|
||||
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = root / result_id
|
||||
if destination.exists():
|
||||
manifest = _read_object(destination / "manifest.json")
|
||||
if (
|
||||
manifest.get("schema_version") != LAB_SCHEMA
|
||||
or manifest.get("identity_sha256") != identity_sha256
|
||||
or manifest.get("result_id") != result_id
|
||||
):
|
||||
raise M48TRiskQualityLabError("existing M4.8T LAB identity conflicts")
|
||||
return M48TRiskQualityLabResult(destination, result_id, manifest)
|
||||
|
||||
temporary = Path(tempfile.mkdtemp(prefix=".m48t-risk-quality-lab-", dir=root))
|
||||
try:
|
||||
(temporary / "review").mkdir(mode=0o700)
|
||||
shutil.copyfile(profile_path, temporary / "profile.json")
|
||||
shutil.copyfile(quality_path, temporary / "worker-quality-result.json")
|
||||
shutil.copyfile(predictions_path, temporary / "predictions.jsonl")
|
||||
shutil.copyfile(failures_path, temporary / "failures.jsonl")
|
||||
shutil.copyfile(temporal_path, temporary / "temporal-semantic-shadow.json")
|
||||
|
||||
review_items: list[dict[str, object]] = []
|
||||
for source in review_paths:
|
||||
destination_image = temporary / "review" / source.name
|
||||
shutil.copyfile(source, destination_image)
|
||||
case_id = source.stem.removeprefix("review-")
|
||||
review_items.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"image_id": int(case_id),
|
||||
"path": f"review/{source.name}",
|
||||
"media_type": "image/jpeg",
|
||||
"byte_length": destination_image.stat().st_size,
|
||||
"sha256": sha256_path(destination_image),
|
||||
}
|
||||
)
|
||||
catalog = {
|
||||
"schema_version": CATALOG_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"case_count": len(review_items),
|
||||
"legend": {
|
||||
"ground_truth": "green",
|
||||
"rf_detr_prediction": "yellow",
|
||||
},
|
||||
"cases": review_items,
|
||||
}
|
||||
catalog_path = temporary / "catalog.json"
|
||||
catalog_path.write_bytes(canonical_json(catalog) + b"\n")
|
||||
|
||||
decision = {
|
||||
"quality_evaluated": True,
|
||||
"quality_accepted": False,
|
||||
"failed_quality_gates": quality["quality_gates"]["failed"],
|
||||
"temporal_invariant_evaluated": True,
|
||||
"temporal_invariant_passed": True,
|
||||
"candidate_accepted": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
limitations = [
|
||||
"COCO val2017 is independent class truth, but it is not RAVNOVES00 domain truth.",
|
||||
"The temporal replay has no independent semantic or physical track-identity truth.",
|
||||
(
|
||||
"Worker timing is throughput evidence for this batch run, not a realtime "
|
||||
"admission gate."
|
||||
),
|
||||
(
|
||||
"Child/adult distinction, unknown moving objects and behavior risk policy "
|
||||
"were not evaluated."
|
||||
),
|
||||
"Semantic labels do not own geometry association, occupancy, navigation or actuation.",
|
||||
]
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"source": {
|
||||
"quality": quality["dataset"],
|
||||
"temporal": temporal["source"],
|
||||
},
|
||||
"configuration": {
|
||||
"candidate": quality["candidate"],
|
||||
"matching": profile["matching"],
|
||||
"quality_gates": profile["quality_gates"],
|
||||
"temporal": profile["temporal"],
|
||||
},
|
||||
"method": method,
|
||||
"execution": quality["execution"],
|
||||
"metrics": {
|
||||
"quality": quality["metrics"],
|
||||
"counts": quality["counts"],
|
||||
"failure_buckets": quality["failure_buckets"],
|
||||
"detector_rejections": quality["detector_rejections"],
|
||||
"temporal": temporal["metrics"],
|
||||
},
|
||||
"acceptance": {
|
||||
"quality": quality["quality_gates"],
|
||||
"temporal_invariant": temporal["temporal_invariant_gate"],
|
||||
},
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
"authority": false_authority(),
|
||||
"visual_evidence": {
|
||||
"kind": "independent-coco-human-truth-review",
|
||||
"case_count": len(review_items),
|
||||
"catalog_schema_version": CATALOG_SCHEMA,
|
||||
"ground_truth_for_ravnoves00": False,
|
||||
},
|
||||
}
|
||||
report_path = temporary / "report.json"
|
||||
report_path.write_bytes(canonical_json(report) + b"\n")
|
||||
artifacts = _artifact_manifest(temporary)
|
||||
manifest = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": created_at_utc,
|
||||
"status": "complete-quality-gate-failed-temporal-invariant-passed",
|
||||
"completed": True,
|
||||
"bounded_question_accepted": False,
|
||||
"ground_truth": False,
|
||||
"catalog": {
|
||||
"path": "catalog.json",
|
||||
"sha256": sha256_path(catalog_path),
|
||||
"byte_length": catalog_path.stat().st_size,
|
||||
},
|
||||
"method": method,
|
||||
"metrics": report["metrics"],
|
||||
"decision": decision,
|
||||
"limitations": limitations,
|
||||
"authority": false_authority(),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
(temporary / "manifest.json").write_bytes(canonical_json(manifest) + b"\n")
|
||||
temporary.replace(destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(temporary, ignore_errors=True)
|
||||
raise
|
||||
return M48TRiskQualityLabResult(destination, result_id, manifest)
|
||||
|
||||
|
||||
def _validate_inputs(
|
||||
*,
|
||||
profile: dict[str, Any],
|
||||
profile_path: Path,
|
||||
quality: dict[str, Any],
|
||||
predictions_path: Path,
|
||||
failures_path: Path,
|
||||
temporal: dict[str, Any],
|
||||
temporal_ledger_path: Path,
|
||||
review_paths: list[Path],
|
||||
) -> None:
|
||||
expected_failed = ["minimum-micro-precision", "minimum-family-recall:vehicle"]
|
||||
if (
|
||||
profile.get("schema_version")
|
||||
!= "missioncore.m48t-risk-quality-temporal-profile/v1"
|
||||
or quality.get("schema_version") != "missioncore.m48t-risk-quality-report/v1"
|
||||
or temporal.get("schema_version")
|
||||
!= "missioncore.m48t-temporal-semantic-shadow/v1"
|
||||
or quality.get("profile_sha256") != sha256_path(profile_path)
|
||||
or temporal.get("profile_sha256") != sha256_path(profile_path)
|
||||
or quality.get("profile_id") != profile.get("profile_id")
|
||||
or temporal.get("profile_id") != profile.get("profile_id")
|
||||
or quality.get("provenance", {}).get("predictions_sha256")
|
||||
!= sha256_path(predictions_path)
|
||||
or quality.get("provenance", {}).get("failures_sha256")
|
||||
!= sha256_path(failures_path)
|
||||
or temporal.get("source", {}).get("frame_ledger_sha256")
|
||||
!= sha256_path(temporal_ledger_path)
|
||||
or quality.get("dataset", {}).get("truth_instances") != 16060
|
||||
or quality.get("quality_gates", {}).get("passed") is not False
|
||||
or quality.get("quality_gates", {}).get("failed") != expected_failed
|
||||
or temporal.get("temporal_invariant_gate", {}).get("passed") is not True
|
||||
or temporal.get("temporal_invariant_gate", {}).get("semantic_quality_accepted")
|
||||
is not False
|
||||
or quality.get("authority", {}).get("candidate_accepted") is not False
|
||||
or len(review_paths) != 16
|
||||
or any(path.is_symlink() or not path.is_file() for path in review_paths)
|
||||
):
|
||||
raise M48TRiskQualityLabError("M4.8T evidence contract changed")
|
||||
|
||||
|
||||
def _method(profile: dict[str, Any], quality: dict[str, Any]) -> dict[str, object]:
|
||||
provenance = quality["provenance"]
|
||||
return {
|
||||
"schema_version": "missioncore.laboratory-method/v1",
|
||||
"completeness": "complete",
|
||||
"execution_class": "hybrid",
|
||||
"pipeline_id": "m48t-coco-quality-plus-bounded-temporal-identity/v1",
|
||||
"components": [
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "COCO 2017 validation",
|
||||
"version": "val2017 independent human annotations",
|
||||
"role": "semantic quality truth",
|
||||
"identity_sha256": provenance["annotations_document_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "model",
|
||||
"name": "RF-DETR-L COCO",
|
||||
"version": quality["candidate"]["model_id"],
|
||||
"role": "fixed-class risk detector under evaluation",
|
||||
"identity_sha256": provenance["runtime_artifact_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": "COCO risk-family scorer",
|
||||
"version": profile["matching"]["method"],
|
||||
"role": "predeclared exact-class and family quality gates",
|
||||
"identity_sha256": provenance["runner_sha256"],
|
||||
},
|
||||
{
|
||||
"kind": "source",
|
||||
"name": "RAVNOVES00 reference-graph replay",
|
||||
"version": "4481-frame immutable publication ledger",
|
||||
"role": "temporal semantic anti-flicker shadow",
|
||||
"identity_sha256": (
|
||||
"badfa2f5f4f33fea7d5ad0e14fe5bbe38e1d490fc661d637789c354b8576d533"
|
||||
),
|
||||
},
|
||||
{
|
||||
"kind": "algorithm",
|
||||
"name": "bounded temporal semantic identity",
|
||||
"version": "history-5-confirm-2-switch-3/v1",
|
||||
"role": "stabilize advisory class on geometry-owned component IDs",
|
||||
"identity_sha256": hashlib.sha256(
|
||||
canonical_json(profile["temporal"])
|
||||
).hexdigest(),
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _artifact_manifest(root: Path) -> list[dict[str, object]]:
|
||||
artifacts: list[dict[str, object]] = []
|
||||
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if relative == "manifest.json":
|
||||
continue
|
||||
media_type = "application/json"
|
||||
schema_version: str | None = None
|
||||
role = "supporting-evidence"
|
||||
if relative == "report.json":
|
||||
role = "laboratory-report"
|
||||
schema_version = REPORT_SCHEMA
|
||||
elif relative == "catalog.json":
|
||||
role = "visual-evidence-catalog"
|
||||
schema_version = CATALOG_SCHEMA
|
||||
elif relative == "profile.json":
|
||||
role = "predeclared-quality-temporal-profile"
|
||||
schema_version = "missioncore.m48t-risk-quality-temporal-profile/v1"
|
||||
elif relative == "worker-quality-result.json":
|
||||
role = "upstream-worker-quality-evidence"
|
||||
schema_version = "missioncore.m48t-risk-quality-report/v1"
|
||||
elif relative == "temporal-semantic-shadow.json":
|
||||
role = "upstream-temporal-shadow-evidence"
|
||||
schema_version = "missioncore.m48t-temporal-semantic-shadow/v1"
|
||||
elif relative.endswith(".jsonl"):
|
||||
media_type = "application/x-ndjson"
|
||||
role = "upstream-quality-ledger"
|
||||
elif relative.endswith(".jpg"):
|
||||
media_type = "image/jpeg"
|
||||
role = "visual-evidence-independent-truth-review"
|
||||
artifacts.append(
|
||||
{
|
||||
"role": role,
|
||||
"path": relative,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": sha256_path(path),
|
||||
"media_type": media_type,
|
||||
"schema_version": schema_version,
|
||||
}
|
||||
)
|
||||
return artifacts
|
||||
|
||||
|
||||
def _read_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text("utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise M48TRiskQualityLabError(f"invalid JSON evidence: {path.name}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise M48TRiskQualityLabError(f"JSON evidence must be an object: {path.name}")
|
||||
return value
|
||||
Reference in New Issue
Block a user