185 lines
6.1 KiB
Python
185 lines
6.1 KiB
Python
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from k1link.compute.e41_methodology_audit import analyze_e41_methodology
|
|
|
|
|
|
def _acceptance(
|
|
item_id: str,
|
|
*,
|
|
split: str,
|
|
frame: int,
|
|
stratum: str = "camera-only",
|
|
presence: str = "object-present",
|
|
) -> dict[str, object]:
|
|
return {
|
|
"item_id": item_id,
|
|
"split": split,
|
|
"source_frame_index": frame,
|
|
"source_stratum": stratum,
|
|
"reference": {
|
|
"presence": presence,
|
|
"geometry_association": "insufficient-support",
|
|
"freshness": "unavailable",
|
|
},
|
|
}
|
|
|
|
|
|
def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]:
|
|
return {
|
|
"item_id": item_id,
|
|
"e29_snapshot": {
|
|
"track_id": track_id,
|
|
},
|
|
}
|
|
|
|
|
|
def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None:
|
|
acceptance = [
|
|
_acceptance("dev-a", split="development", frame=100, presence="object-present"),
|
|
_acceptance(
|
|
"dev-b",
|
|
split="development",
|
|
frame=101,
|
|
presence="background-or-noise",
|
|
),
|
|
_acceptance("val-a", split="validation", frame=100, presence="object-present"),
|
|
_acceptance("val-b", split="validation", frame=149, presence="object-present"),
|
|
]
|
|
materialization = [
|
|
_materialization("dev-a", track_id=7),
|
|
_materialization("dev-b", track_id=8),
|
|
_materialization("val-a", track_id=7),
|
|
_materialization("val-b", track_id=None),
|
|
]
|
|
names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"]
|
|
matrix = np.asarray(
|
|
[
|
|
[0.1, 1.0, 100.0],
|
|
[0.9, 1.0, 101.0],
|
|
[0.2, 1.0, 100.0],
|
|
[0.3, 1.0, 149.0],
|
|
],
|
|
dtype=np.float64,
|
|
)
|
|
report = analyze_e41_methodology(
|
|
acceptance_rows=acceptance,
|
|
materialization_rows=materialization,
|
|
feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"],
|
|
feature_names=names,
|
|
feature_matrix=matrix,
|
|
e40_model={
|
|
"feature_names": names,
|
|
"camera_only_training_items": 2,
|
|
"dimension_projection": "source-stratum-plus-presence/v1",
|
|
},
|
|
e40_report={
|
|
"status": "measured-leakage-resistant-product-gate",
|
|
"execution": {"class": "sealed-validation-evaluation"},
|
|
"metrics": {
|
|
"dimensions": {
|
|
"presence": {"accuracy": 0.5},
|
|
"geometry_association": {"accuracy": 0.5},
|
|
}
|
|
},
|
|
},
|
|
e40_predictions=[
|
|
{
|
|
"item_id": "val-a",
|
|
"prediction": {"presence": "object-present"},
|
|
"reference": {"presence": "object-present"},
|
|
"scored": True,
|
|
}
|
|
],
|
|
label_provenance={
|
|
"engineering_items": 4,
|
|
"human_exception_items": 0,
|
|
"independent_ground_truth": False,
|
|
},
|
|
time_block_frames=50,
|
|
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
|
)
|
|
|
|
assert report["split_leakage"]["exact_source_frames"]["count"] == 1
|
|
assert report["split_leakage"]["track_ids"]["count"] == 1
|
|
assert report["split_leakage"]["time_blocks"]["count"] == 1
|
|
assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1
|
|
assert report["features"]["camera_only_training_items"] == 2
|
|
assert report["features"]["forbidden_features"] == ["source_frame_index"]
|
|
assert report["predictor_evaluator_boundary"]["physically_separated"] is False
|
|
assert report["metric_semantics"]["dimensions_independently_inferred"] is False
|
|
assert report["policy"]["blind_gate_eligible"] is False
|
|
assert report["policy"]["violations"] == [
|
|
"labels-are-not-independent-ground-truth",
|
|
"development-validation-source-groups-overlap",
|
|
"prediction-and-evaluation-concerns-are-co-located",
|
|
"historical-e40-still-contains-blind-or-product-gate-claims",
|
|
"forbidden-identity-feature-detected",
|
|
]
|
|
|
|
|
|
def test_e41_accepts_a_clean_separated_methodology_contract() -> None:
|
|
acceptance = [
|
|
_acceptance("dev-a", split="development", frame=10),
|
|
_acceptance(
|
|
"dev-b",
|
|
split="development",
|
|
frame=11,
|
|
presence="background-or-noise",
|
|
),
|
|
_acceptance("val-a", split="validation", frame=210),
|
|
]
|
|
materialization = [
|
|
_materialization("dev-a", track_id=1),
|
|
_materialization("dev-b", track_id=2),
|
|
_materialization("val-a", track_id=9),
|
|
]
|
|
names = ["image_luma_mean", "support_occupied_fraction"]
|
|
report = analyze_e41_methodology(
|
|
acceptance_rows=acceptance,
|
|
materialization_rows=materialization,
|
|
feature_item_ids=["dev-a", "dev-b", "val-a"],
|
|
feature_names=names,
|
|
feature_matrix=np.asarray(
|
|
[
|
|
[0.1, 0.2],
|
|
[0.9, 0.8],
|
|
[0.4, 0.3],
|
|
],
|
|
dtype=np.float64,
|
|
),
|
|
e40_model={
|
|
"feature_names": names,
|
|
"camera_only_training_items": 2,
|
|
"dimension_projection": "independent-task-heads/v1",
|
|
},
|
|
e40_report={
|
|
"status": "source-scoped-visible-evaluation",
|
|
"metrics": {
|
|
"dimensions": {
|
|
"presence": {"accuracy": 0.8},
|
|
"geometry_association": {"accuracy": 0.7},
|
|
}
|
|
},
|
|
},
|
|
e40_predictions=[
|
|
{
|
|
"item_id": "val-a",
|
|
"prediction": {"presence": "object-present"},
|
|
}
|
|
],
|
|
label_provenance={
|
|
"engineering_items": 0,
|
|
"human_exception_items": 3,
|
|
"independent_ground_truth": True,
|
|
},
|
|
time_block_frames=50,
|
|
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
|
)
|
|
|
|
assert report["features"]["forbidden_features"] == []
|
|
assert report["predictor_evaluator_boundary"]["physically_separated"] is True
|
|
assert report["policy"]["violations"] == []
|
|
assert report["policy"]["blind_gate_eligible"] is True
|