feat(perception): stabilize pre-capture methodology
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.compute.e41_methodology_audit import analyze_e41_methodology
|
||||
|
||||
|
||||
def _acceptance(
|
||||
item_id: str,
|
||||
*,
|
||||
split: str,
|
||||
frame: int,
|
||||
stratum: str = "camera-only",
|
||||
presence: str = "object-present",
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": item_id,
|
||||
"split": split,
|
||||
"source_frame_index": frame,
|
||||
"source_stratum": stratum,
|
||||
"reference": {
|
||||
"presence": presence,
|
||||
"geometry_association": "insufficient-support",
|
||||
"freshness": "unavailable",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": item_id,
|
||||
"e29_snapshot": {
|
||||
"track_id": track_id,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None:
|
||||
acceptance = [
|
||||
_acceptance("dev-a", split="development", frame=100, presence="object-present"),
|
||||
_acceptance(
|
||||
"dev-b",
|
||||
split="development",
|
||||
frame=101,
|
||||
presence="background-or-noise",
|
||||
),
|
||||
_acceptance("val-a", split="validation", frame=100, presence="object-present"),
|
||||
_acceptance("val-b", split="validation", frame=149, presence="object-present"),
|
||||
]
|
||||
materialization = [
|
||||
_materialization("dev-a", track_id=7),
|
||||
_materialization("dev-b", track_id=8),
|
||||
_materialization("val-a", track_id=7),
|
||||
_materialization("val-b", track_id=None),
|
||||
]
|
||||
names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"]
|
||||
matrix = np.asarray(
|
||||
[
|
||||
[0.1, 1.0, 100.0],
|
||||
[0.9, 1.0, 101.0],
|
||||
[0.2, 1.0, 100.0],
|
||||
[0.3, 1.0, 149.0],
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
report = analyze_e41_methodology(
|
||||
acceptance_rows=acceptance,
|
||||
materialization_rows=materialization,
|
||||
feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"],
|
||||
feature_names=names,
|
||||
feature_matrix=matrix,
|
||||
e40_model={
|
||||
"feature_names": names,
|
||||
"camera_only_training_items": 2,
|
||||
"dimension_projection": "source-stratum-plus-presence/v1",
|
||||
},
|
||||
e40_report={
|
||||
"status": "measured-leakage-resistant-product-gate",
|
||||
"execution": {"class": "sealed-validation-evaluation"},
|
||||
"metrics": {
|
||||
"dimensions": {
|
||||
"presence": {"accuracy": 0.5},
|
||||
"geometry_association": {"accuracy": 0.5},
|
||||
}
|
||||
},
|
||||
},
|
||||
e40_predictions=[
|
||||
{
|
||||
"item_id": "val-a",
|
||||
"prediction": {"presence": "object-present"},
|
||||
"reference": {"presence": "object-present"},
|
||||
"scored": True,
|
||||
}
|
||||
],
|
||||
label_provenance={
|
||||
"engineering_items": 4,
|
||||
"human_exception_items": 0,
|
||||
"independent_ground_truth": False,
|
||||
},
|
||||
time_block_frames=50,
|
||||
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
||||
)
|
||||
|
||||
assert report["split_leakage"]["exact_source_frames"]["count"] == 1
|
||||
assert report["split_leakage"]["track_ids"]["count"] == 1
|
||||
assert report["split_leakage"]["time_blocks"]["count"] == 1
|
||||
assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1
|
||||
assert report["features"]["camera_only_training_items"] == 2
|
||||
assert report["features"]["forbidden_features"] == ["source_frame_index"]
|
||||
assert report["predictor_evaluator_boundary"]["physically_separated"] is False
|
||||
assert report["metric_semantics"]["dimensions_independently_inferred"] is False
|
||||
assert report["policy"]["blind_gate_eligible"] is False
|
||||
assert report["policy"]["violations"] == [
|
||||
"labels-are-not-independent-ground-truth",
|
||||
"development-validation-source-groups-overlap",
|
||||
"prediction-and-evaluation-concerns-are-co-located",
|
||||
"historical-e40-still-contains-blind-or-product-gate-claims",
|
||||
"forbidden-identity-feature-detected",
|
||||
]
|
||||
|
||||
|
||||
def test_e41_accepts_a_clean_separated_methodology_contract() -> None:
|
||||
acceptance = [
|
||||
_acceptance("dev-a", split="development", frame=10),
|
||||
_acceptance(
|
||||
"dev-b",
|
||||
split="development",
|
||||
frame=11,
|
||||
presence="background-or-noise",
|
||||
),
|
||||
_acceptance("val-a", split="validation", frame=210),
|
||||
]
|
||||
materialization = [
|
||||
_materialization("dev-a", track_id=1),
|
||||
_materialization("dev-b", track_id=2),
|
||||
_materialization("val-a", track_id=9),
|
||||
]
|
||||
names = ["image_luma_mean", "support_occupied_fraction"]
|
||||
report = analyze_e41_methodology(
|
||||
acceptance_rows=acceptance,
|
||||
materialization_rows=materialization,
|
||||
feature_item_ids=["dev-a", "dev-b", "val-a"],
|
||||
feature_names=names,
|
||||
feature_matrix=np.asarray(
|
||||
[
|
||||
[0.1, 0.2],
|
||||
[0.9, 0.8],
|
||||
[0.4, 0.3],
|
||||
],
|
||||
dtype=np.float64,
|
||||
),
|
||||
e40_model={
|
||||
"feature_names": names,
|
||||
"camera_only_training_items": 2,
|
||||
"dimension_projection": "independent-task-heads/v1",
|
||||
},
|
||||
e40_report={
|
||||
"status": "source-scoped-visible-evaluation",
|
||||
"metrics": {
|
||||
"dimensions": {
|
||||
"presence": {"accuracy": 0.8},
|
||||
"geometry_association": {"accuracy": 0.7},
|
||||
}
|
||||
},
|
||||
},
|
||||
e40_predictions=[
|
||||
{
|
||||
"item_id": "val-a",
|
||||
"prediction": {"presence": "object-present"},
|
||||
}
|
||||
],
|
||||
label_provenance={
|
||||
"engineering_items": 0,
|
||||
"human_exception_items": 3,
|
||||
"independent_ground_truth": True,
|
||||
},
|
||||
time_block_frames=50,
|
||||
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
||||
)
|
||||
|
||||
assert report["features"]["forbidden_features"] == []
|
||||
assert report["predictor_evaluator_boundary"]["physically_separated"] is True
|
||||
assert report["policy"]["violations"] == []
|
||||
assert report["policy"]["blind_gate_eligible"] is True
|
||||
Reference in New Issue
Block a user