from __future__ import annotations import numpy as np from k1link.compute.e41_methodology_audit import analyze_e41_methodology def _acceptance( item_id: str, *, split: str, frame: int, stratum: str = "camera-only", presence: str = "object-present", ) -> dict[str, object]: return { "item_id": item_id, "split": split, "source_frame_index": frame, "source_stratum": stratum, "reference": { "presence": presence, "geometry_association": "insufficient-support", "freshness": "unavailable", }, } def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]: return { "item_id": item_id, "e29_snapshot": { "track_id": track_id, }, } def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None: acceptance = [ _acceptance("dev-a", split="development", frame=100, presence="object-present"), _acceptance( "dev-b", split="development", frame=101, presence="background-or-noise", ), _acceptance("val-a", split="validation", frame=100, presence="object-present"), _acceptance("val-b", split="validation", frame=149, presence="object-present"), ] materialization = [ _materialization("dev-a", track_id=7), _materialization("dev-b", track_id=8), _materialization("val-a", track_id=7), _materialization("val-b", track_id=None), ] names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"] matrix = np.asarray( [ [0.1, 1.0, 100.0], [0.9, 1.0, 101.0], [0.2, 1.0, 100.0], [0.3, 1.0, 149.0], ], dtype=np.float64, ) report = analyze_e41_methodology( acceptance_rows=acceptance, materialization_rows=materialization, feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"], feature_names=names, feature_matrix=matrix, e40_model={ "feature_names": names, "camera_only_training_items": 2, "dimension_projection": "source-stratum-plus-presence/v1", }, e40_report={ "status": "measured-leakage-resistant-product-gate", "execution": {"class": "sealed-validation-evaluation"}, "metrics": { "dimensions": { "presence": {"accuracy": 0.5}, "geometry_association": {"accuracy": 0.5}, } }, }, e40_predictions=[ { "item_id": "val-a", "prediction": {"presence": "object-present"}, "reference": {"presence": "object-present"}, "scored": True, } ], label_provenance={ "engineering_items": 4, "human_exception_items": 0, "independent_ground_truth": False, }, time_block_frames=50, forbidden_feature_tokens=("source_frame", "track_id", "path"), ) assert report["split_leakage"]["exact_source_frames"]["count"] == 1 assert report["split_leakage"]["track_ids"]["count"] == 1 assert report["split_leakage"]["time_blocks"]["count"] == 1 assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1 assert report["features"]["camera_only_training_items"] == 2 assert report["features"]["forbidden_features"] == ["source_frame_index"] assert report["predictor_evaluator_boundary"]["physically_separated"] is False assert report["metric_semantics"]["dimensions_independently_inferred"] is False assert report["policy"]["blind_gate_eligible"] is False assert report["policy"]["violations"] == [ "labels-are-not-independent-ground-truth", "development-validation-source-groups-overlap", "prediction-and-evaluation-concerns-are-co-located", "historical-e40-still-contains-blind-or-product-gate-claims", "forbidden-identity-feature-detected", ] def test_e41_accepts_a_clean_separated_methodology_contract() -> None: acceptance = [ _acceptance("dev-a", split="development", frame=10), _acceptance( "dev-b", split="development", frame=11, presence="background-or-noise", ), _acceptance("val-a", split="validation", frame=210), ] materialization = [ _materialization("dev-a", track_id=1), _materialization("dev-b", track_id=2), _materialization("val-a", track_id=9), ] names = ["image_luma_mean", "support_occupied_fraction"] report = analyze_e41_methodology( acceptance_rows=acceptance, materialization_rows=materialization, feature_item_ids=["dev-a", "dev-b", "val-a"], feature_names=names, feature_matrix=np.asarray( [ [0.1, 0.2], [0.9, 0.8], [0.4, 0.3], ], dtype=np.float64, ), e40_model={ "feature_names": names, "camera_only_training_items": 2, "dimension_projection": "independent-task-heads/v1", }, e40_report={ "status": "source-scoped-visible-evaluation", "metrics": { "dimensions": { "presence": {"accuracy": 0.8}, "geometry_association": {"accuracy": 0.7}, } }, }, e40_predictions=[ { "item_id": "val-a", "prediction": {"presence": "object-present"}, } ], label_provenance={ "engineering_items": 0, "human_exception_items": 3, "independent_ground_truth": True, }, time_block_frames=50, forbidden_feature_tokens=("source_frame", "track_id", "path"), ) assert report["features"]["forbidden_features"] == [] assert report["predictor_evaluator_boundary"]["physically_separated"] is True assert report["policy"]["violations"] == [] assert report["policy"]["blind_gate_eligible"] is True