feat(lab): measure RAVNOVES00 R1 quality baseline
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,830 @@
|
||||
"""Development-trained RAVNOVES00 R1 perception-quality baseline.
|
||||
|
||||
E38 consumes the frozen E37 contract without changing its denominator or
|
||||
validation split. A small deterministic decision tree is fitted only on the
|
||||
development partition and is then evaluated once on the sealed validation
|
||||
partition. The result remains source-scoped and diagnostic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from k1link.compute.e37_acceptance_contract import (
|
||||
E37_ITEMS_NAME,
|
||||
E37AcceptanceContractError,
|
||||
read_e37_acceptance_contract,
|
||||
)
|
||||
|
||||
E38_PROFILE_SCHEMA: Final = "missioncore.e38-perception-baseline-profile/v1"
|
||||
E38_PACKAGE_SCHEMA: Final = "missioncore.e38-worker-package/v1"
|
||||
E38_RESULT_SCHEMA: Final = "missioncore.e38-perception-baseline/v1"
|
||||
E38_PREDICTION_SCHEMA: Final = "missioncore.e38-perception-prediction/v1"
|
||||
E38_MODEL_SCHEMA: Final = "missioncore.e38-development-cart-model/v1"
|
||||
E38_REPORT_SCHEMA: Final = "missioncore.e38-perception-baseline-report/v1"
|
||||
|
||||
E38_PREDICTIONS_NAME: Final = "predictions.jsonl"
|
||||
E38_MODEL_NAME: Final = "development-model.json"
|
||||
E38_REPORT_NAME: Final = "run-report.json"
|
||||
E38_MANIFEST_NAME: Final = "manifest.json"
|
||||
|
||||
_MATERIALIZATION_SCHEMA: Final = "missioncore.e30-evidence-materialization/v2"
|
||||
_SOURCE_SESSION_ID: Final = "20260720T065719Z_viewer_live"
|
||||
_SOURCE_DISPLAY_NAME: Final = "RAVNOVES00"
|
||||
_DIMENSIONS: Final = ("presence", "geometry_association", "freshness")
|
||||
_CATEGORICAL_FEATURES: Final = {
|
||||
"stratum": ("agree", "camera-only", "conflict", "geometry-only", "unknown"),
|
||||
"range": ("near", "middle", "far", "unavailable"),
|
||||
"geometry": (
|
||||
"agree",
|
||||
"conflict",
|
||||
"single-source-camera",
|
||||
"single-source-geometry",
|
||||
"unknown",
|
||||
"unavailable",
|
||||
),
|
||||
}
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
|
||||
|
||||
class E38PerceptionBaselineError(RuntimeError):
|
||||
"""The E38 profile, immutable inputs, or result is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class E38PerceptionBaseline:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
manifest: dict[str, Any]
|
||||
report: dict[str, Any]
|
||||
model: dict[str, Any]
|
||||
|
||||
@property
|
||||
def quality_gate_passed(self) -> bool:
|
||||
return self.report.get("quality_gate", {}).get("passed") is True
|
||||
|
||||
|
||||
def build_e38_perception_baseline(
|
||||
*,
|
||||
acceptance_root: Path,
|
||||
materialization_root: Path,
|
||||
profile_path: Path,
|
||||
output_root: Path,
|
||||
worker_node: str | None = None,
|
||||
) -> E38PerceptionBaseline:
|
||||
"""Fit on E37 development rows and evaluate the sealed validation rows."""
|
||||
|
||||
profile_file = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_file)
|
||||
_validate_profile(profile)
|
||||
acceptance_path = acceptance_root.resolve(strict=True)
|
||||
materialization_path = materialization_root.resolve(strict=True)
|
||||
try:
|
||||
acceptance = read_e37_acceptance_contract(acceptance_path)
|
||||
except E37AcceptanceContractError as exc:
|
||||
raise E38PerceptionBaselineError("E38 E37 contract is invalid") from exc
|
||||
if acceptance.result_id != profile["source"]["acceptance_result_id"]:
|
||||
raise E38PerceptionBaselineError("E38 acceptance identity changed")
|
||||
|
||||
acceptance_items_path = acceptance_path / E37_ITEMS_NAME
|
||||
acceptance_rows = _read_jsonl(acceptance_items_path)
|
||||
materialization_rows, materialization_binding = _load_materialization(
|
||||
materialization_path,
|
||||
acceptance.manifest,
|
||||
profile,
|
||||
)
|
||||
if (
|
||||
len(acceptance_rows) != 486
|
||||
or len(materialization_rows) != 486
|
||||
or {row.get("item_id") for row in acceptance_rows}
|
||||
!= {row.get("item_id") for row in materialization_rows}
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 denominator accounting differs")
|
||||
|
||||
identity = {
|
||||
"schema_version": E38_RESULT_SCHEMA,
|
||||
"source": {
|
||||
"session_id": _SOURCE_SESSION_ID,
|
||||
"display_name": _SOURCE_DISPLAY_NAME,
|
||||
"acceptance_result_id": acceptance.result_id,
|
||||
"acceptance_identity_sha256": acceptance.manifest["identity_sha256"],
|
||||
"acceptance_items_sha256": _sha256(acceptance_items_path),
|
||||
**materialization_binding,
|
||||
},
|
||||
"profile": {
|
||||
"profile_id": profile["profile_id"],
|
||||
"sha256": _sha256(profile_file),
|
||||
},
|
||||
"execution": {
|
||||
"class": "deterministic-development-trained-validation-evaluation",
|
||||
"worker_node": worker_node or "unbound-local",
|
||||
},
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e38-perception-baseline-{identity_sha256}"
|
||||
destination = output_root.expanduser().absolute() / result_id
|
||||
if destination.exists():
|
||||
return read_e38_perception_baseline(destination)
|
||||
|
||||
materialization_by_id = {
|
||||
str(row["item_id"]): row for row in materialization_rows
|
||||
}
|
||||
joined: list[dict[str, Any]] = []
|
||||
for acceptance_row in acceptance_rows:
|
||||
item_id = str(acceptance_row.get("item_id"))
|
||||
reference = acceptance_row.get("reference")
|
||||
if (
|
||||
acceptance_row.get("split") not in {"development", "validation"}
|
||||
or acceptance_row.get("severity") not in {"standard", "medium", "high"}
|
||||
or not isinstance(reference, dict)
|
||||
or set(reference) != set(_DIMENSIONS)
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 acceptance row is invalid")
|
||||
joined.append(
|
||||
{
|
||||
"acceptance": acceptance_row,
|
||||
"features": _feature_vector(
|
||||
acceptance_row,
|
||||
materialization_by_id[item_id],
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
model_dimensions: dict[str, Any] = {}
|
||||
for dimension in _DIMENSIONS:
|
||||
dimension_profile = profile["model"]["dimensions"][dimension]
|
||||
feature_names = _dimension_features(
|
||||
joined,
|
||||
include_source_time=dimension_profile["include_source_time"],
|
||||
)
|
||||
development = [
|
||||
(
|
||||
_selected_features(row["features"], feature_names),
|
||||
str(row["acceptance"]["reference"][dimension]),
|
||||
)
|
||||
for row in joined
|
||||
if row["acceptance"]["split"] == "development"
|
||||
]
|
||||
tree = _train_tree(
|
||||
development,
|
||||
feature_names=feature_names,
|
||||
max_depth=int(dimension_profile["max_depth"]),
|
||||
min_leaf=int(dimension_profile["min_leaf"]),
|
||||
)
|
||||
model_dimensions[dimension] = {
|
||||
"feature_names": feature_names,
|
||||
"max_depth": dimension_profile["max_depth"],
|
||||
"min_leaf": dimension_profile["min_leaf"],
|
||||
"tree": tree,
|
||||
}
|
||||
|
||||
predictions: list[dict[str, Any]] = []
|
||||
for row in joined:
|
||||
acceptance_row = row["acceptance"]
|
||||
predicted = {
|
||||
dimension: _predict_tree(
|
||||
model_dimensions[dimension]["tree"],
|
||||
row["features"],
|
||||
)
|
||||
for dimension in _DIMENSIONS
|
||||
}
|
||||
predictions.append(
|
||||
{
|
||||
"schema_version": E38_PREDICTION_SCHEMA,
|
||||
"sequence": acceptance_row["sequence"],
|
||||
"item_id": acceptance_row["item_id"],
|
||||
"review_key": acceptance_row["review_key"],
|
||||
"source_frame_index": acceptance_row["source_frame_index"],
|
||||
"source_stratum": acceptance_row["source_stratum"],
|
||||
"severity": acceptance_row["severity"],
|
||||
"split": acceptance_row["split"],
|
||||
"prediction": predicted,
|
||||
"reference": acceptance_row["reference"],
|
||||
"scored": acceptance_row["split"] == "validation",
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
)
|
||||
|
||||
validation_predictions = [
|
||||
row for row in predictions if row["split"] == "validation"
|
||||
]
|
||||
development_count = sum(row["split"] == "development" for row in predictions)
|
||||
validation_count = len(validation_predictions)
|
||||
if development_count != 340 or validation_count != 146:
|
||||
raise E38PerceptionBaselineError("E38 frozen split changed")
|
||||
dimension_metrics = {
|
||||
dimension: _dimension_metrics(
|
||||
validation_predictions,
|
||||
dimension=dimension,
|
||||
target=float(profile["targets"][f"{dimension}_target"]),
|
||||
)
|
||||
for dimension in _DIMENSIONS
|
||||
}
|
||||
high_severity_failures = sum(
|
||||
row["severity"] == "high"
|
||||
and any(
|
||||
row["prediction"][dimension] != row["reference"][dimension]
|
||||
for dimension in _DIMENSIONS
|
||||
)
|
||||
for row in validation_predictions
|
||||
)
|
||||
accounting_fraction = (
|
||||
len(validation_predictions) / validation_count if validation_count else 0.0
|
||||
)
|
||||
false_free_claims = sum(
|
||||
value == "free"
|
||||
for row in predictions
|
||||
for value in row["prediction"].values()
|
||||
)
|
||||
gate_checks = {
|
||||
"presence_target_reached": dimension_metrics["presence"]["passed"],
|
||||
"geometry_association_target_reached": dimension_metrics[
|
||||
"geometry_association"
|
||||
]["passed"],
|
||||
"freshness_target_reached": dimension_metrics["freshness"]["passed"],
|
||||
"validation_accounting_complete": math.isclose(accounting_fraction, 1.0),
|
||||
"false_free_claims_zero": false_free_claims == 0,
|
||||
"high_severity_failures_zero": high_severity_failures == 0,
|
||||
"authority_remains_diagnostic": True,
|
||||
}
|
||||
quality_gate_passed = all(gate_checks.values())
|
||||
model_document = {
|
||||
"schema_version": E38_MODEL_SCHEMA,
|
||||
"profile_id": profile["profile_id"],
|
||||
"training_split": "development",
|
||||
"training_items": development_count,
|
||||
"validation_labels_used_for_training": False,
|
||||
"dimensions": model_dimensions,
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
report = {
|
||||
"schema_version": E38_REPORT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"status": "measured-r1-source-scoped-baseline",
|
||||
"source_session_id": _SOURCE_SESSION_ID,
|
||||
"source_display_name": _SOURCE_DISPLAY_NAME,
|
||||
"profile_id": profile["profile_id"],
|
||||
"execution": identity["execution"],
|
||||
"metrics": {
|
||||
"development_items": development_count,
|
||||
"validation_items": validation_count,
|
||||
"terminal_outcomes": validation_count,
|
||||
"accounting_fraction": round(accounting_fraction, 6),
|
||||
"false_free_claims": false_free_claims,
|
||||
"high_severity_failures": high_severity_failures,
|
||||
"dimensions": dimension_metrics,
|
||||
},
|
||||
"quality_gate": {
|
||||
"passed": quality_gate_passed,
|
||||
"checks": gate_checks,
|
||||
"blocking_checks": [
|
||||
name for name, passed in gate_checks.items() if not passed
|
||||
],
|
||||
},
|
||||
"decision": {
|
||||
"r1_baseline_measured": True,
|
||||
"accepted_for_release": quality_gate_passed,
|
||||
"next_gate": (
|
||||
"R2 source-scoped temporal product state"
|
||||
if quality_gate_passed
|
||||
else "R1 detector presence and geometry-association improvement"
|
||||
),
|
||||
},
|
||||
"limitations": [
|
||||
(
|
||||
"the model is trained and evaluated only on the source-scoped "
|
||||
"RAVNOVES00 engineering-reviewed contract"
|
||||
),
|
||||
"validation labels are evaluation-only and never used to fit a tree",
|
||||
(
|
||||
"the result is not independent ground truth and does not prove "
|
||||
"another route, camera, rig or mount"
|
||||
),
|
||||
"navigation, command and safety authority remain false",
|
||||
],
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
staging = destination.parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
_write_jsonl(staging / E38_PREDICTIONS_NAME, predictions)
|
||||
_write_json(staging / E38_MODEL_NAME, model_document)
|
||||
_write_json(staging / E38_REPORT_NAME, report)
|
||||
artifacts = [
|
||||
_artifact(staging / E38_PREDICTIONS_NAME, "sealed-evaluation"),
|
||||
_artifact(staging / E38_MODEL_NAME, "development-trained-model"),
|
||||
_artifact(staging / E38_REPORT_NAME, "quality-report"),
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": E38_RESULT_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": _utc_now(),
|
||||
"acceptance_state": "completed-r1-source-scoped-baseline",
|
||||
"quality_gate_passed": quality_gate_passed,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(staging / E38_MANIFEST_NAME, manifest)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return read_e38_perception_baseline(destination)
|
||||
|
||||
|
||||
def read_e38_perception_baseline(root: Path) -> E38PerceptionBaseline:
|
||||
"""Validate and read one immutable E38 result."""
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / E38_MANIFEST_NAME)
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
manifest.get("schema_version") != E38_RESULT_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or manifest.get("result_id") != f"e38-perception-baseline-{identity_sha256}"
|
||||
or resolved.name != manifest.get("result_id")
|
||||
or manifest.get("acceptance_state")
|
||||
!= "completed-r1-source-scoped-baseline"
|
||||
or identity.get("authority") != _AUTHORITY
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 result identity is invalid")
|
||||
expected = {
|
||||
E38_PREDICTIONS_NAME: "sealed-evaluation",
|
||||
E38_MODEL_NAME: "development-trained-model",
|
||||
E38_REPORT_NAME: "quality-report",
|
||||
}
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list) or len(artifacts) != len(expected):
|
||||
raise E38PerceptionBaselineError("E38 artifact catalog is invalid")
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E38PerceptionBaselineError("E38 artifact descriptor is invalid")
|
||||
name = row.get("path")
|
||||
path = resolved / str(name)
|
||||
if (
|
||||
name not in expected
|
||||
or row.get("role") != expected[name]
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 artifact content changed")
|
||||
report = _read_json(resolved / E38_REPORT_NAME)
|
||||
model = _read_json(resolved / E38_MODEL_NAME)
|
||||
if (
|
||||
report.get("schema_version") != E38_REPORT_SCHEMA
|
||||
or report.get("result_id") != resolved.name
|
||||
or report.get("identity_sha256") != identity_sha256
|
||||
or report.get("status") != "measured-r1-source-scoped-baseline"
|
||||
or model.get("schema_version") != E38_MODEL_SCHEMA
|
||||
or model.get("validation_labels_used_for_training") is not False
|
||||
or manifest.get("quality_gate_passed")
|
||||
is not report.get("quality_gate", {}).get("passed")
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 report or model is invalid")
|
||||
return E38PerceptionBaseline(
|
||||
result_id=resolved.name,
|
||||
result_root=resolved,
|
||||
manifest=manifest,
|
||||
report=report,
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
def _load_materialization(
|
||||
root: Path,
|
||||
acceptance_manifest: dict[str, Any],
|
||||
profile: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
manifest_path = root / "manifest.json"
|
||||
index_path = root / "materialized-items.jsonl"
|
||||
manifest = _read_json(manifest_path)
|
||||
binding = (
|
||||
acceptance_manifest.get("identity", {})
|
||||
.get("reviewed_substrate", {})
|
||||
)
|
||||
if (
|
||||
manifest.get("schema_version") != _MATERIALIZATION_SCHEMA
|
||||
or root.name != profile["source"]["materialization_id"]
|
||||
or manifest.get("result_id") != root.name
|
||||
or root.name != binding.get("materialization_id")
|
||||
or manifest.get("identity_sha256")
|
||||
!= binding.get("materialization_identity_sha256")
|
||||
or _sha256(manifest_path)
|
||||
!= binding.get("materialization_manifest_sha256")
|
||||
or not index_path.is_file()
|
||||
or index_path.is_symlink()
|
||||
or _sha256(index_path) != binding.get("materialization_index_sha256")
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 materialization identity changed")
|
||||
rows = _read_jsonl(index_path)
|
||||
return rows, {
|
||||
"materialization_id": root.name,
|
||||
"materialization_identity_sha256": manifest["identity_sha256"],
|
||||
"materialization_index_sha256": _sha256(index_path),
|
||||
}
|
||||
|
||||
|
||||
def _feature_vector(
|
||||
acceptance: dict[str, Any],
|
||||
materialization: dict[str, Any],
|
||||
) -> dict[str, float]:
|
||||
snapshot = _object(materialization.get("e29_snapshot"), "E38 snapshot")
|
||||
evidence = _object(
|
||||
materialization.get("materialization"),
|
||||
"E38 materialization evidence",
|
||||
)
|
||||
bounds = snapshot.get("bounds_map_xyz_m")
|
||||
height = snapshot.get("height_range_m")
|
||||
features = {
|
||||
"detector_score": _number_or(evidence.get("detector_score"), -1.0),
|
||||
"selected_points": _number_or(evidence.get("selected_point_count"), 0.0),
|
||||
"rejected_points": _number_or(
|
||||
evidence.get("rejected_candidate_point_count"),
|
||||
0.0,
|
||||
),
|
||||
"candidate_points": _number_or(evidence.get("candidate_point_count"), 0.0),
|
||||
"camera_front_points": _number_or(
|
||||
evidence.get("camera_front_point_count"),
|
||||
0.0,
|
||||
),
|
||||
"projected_points": _number_or(
|
||||
evidence.get("projected_point_count"),
|
||||
0.0,
|
||||
),
|
||||
"frame_points": _number_or(evidence.get("frame_point_count"), 0.0),
|
||||
"point_count": _number_or(snapshot.get("point_count"), 0.0),
|
||||
"voxel_count": _number_or(snapshot.get("voxel_count"), 0.0),
|
||||
"nearest_range_m": _number_or(snapshot.get("nearest_range_m"), -1.0),
|
||||
"height_span_m": _span(height, 0),
|
||||
"bounds_span_x_m": _span(bounds, 0),
|
||||
"bounds_span_y_m": _span(bounds, 1),
|
||||
"bounds_span_z_m": _span(bounds, 2),
|
||||
"session_seconds": _number_or(acceptance.get("session_seconds"), -1.0),
|
||||
"source_frame_index": _number_or(
|
||||
acceptance.get("source_frame_index"),
|
||||
-1.0,
|
||||
),
|
||||
}
|
||||
categorical = {
|
||||
"stratum": materialization.get("stratum"),
|
||||
"range": materialization.get("range_bucket"),
|
||||
"geometry": snapshot.get("geometry_status"),
|
||||
}
|
||||
for prefix, values in _CATEGORICAL_FEATURES.items():
|
||||
observed = str(categorical[prefix])
|
||||
for value in values:
|
||||
features[f"{prefix}={value}"] = float(observed == value)
|
||||
return features
|
||||
|
||||
|
||||
def _dimension_features(
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
include_source_time: bool,
|
||||
) -> list[str]:
|
||||
features = sorted(
|
||||
{
|
||||
name
|
||||
for row in rows
|
||||
for name in row["features"]
|
||||
if include_source_time
|
||||
or name not in {"session_seconds", "source_frame_index"}
|
||||
}
|
||||
)
|
||||
if not features:
|
||||
raise E38PerceptionBaselineError("E38 feature set is empty")
|
||||
return features
|
||||
|
||||
|
||||
def _selected_features(
|
||||
values: dict[str, float],
|
||||
names: list[str],
|
||||
) -> dict[str, float]:
|
||||
return {name: values.get(name, 0.0) for name in names}
|
||||
|
||||
|
||||
def _train_tree(
|
||||
rows: list[tuple[dict[str, float], str]],
|
||||
*,
|
||||
feature_names: list[str],
|
||||
max_depth: int,
|
||||
min_leaf: int,
|
||||
depth: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
if not rows:
|
||||
raise E38PerceptionBaselineError("E38 tree has no training rows")
|
||||
labels = [label for _, label in rows]
|
||||
prediction = sorted(Counter(labels).items(), key=lambda item: (-item[1], item[0]))[
|
||||
0
|
||||
][0]
|
||||
node: dict[str, Any] = {
|
||||
"prediction": prediction,
|
||||
"samples": len(rows),
|
||||
"distribution": dict(sorted(Counter(labels).items())),
|
||||
}
|
||||
if (
|
||||
depth >= max_depth
|
||||
or len(set(labels)) == 1
|
||||
or len(rows) < 2 * min_leaf
|
||||
):
|
||||
return node
|
||||
base_impurity = _gini(labels)
|
||||
best: tuple[
|
||||
float,
|
||||
str,
|
||||
float,
|
||||
list[tuple[dict[str, float], str]],
|
||||
list[tuple[dict[str, float], str]],
|
||||
] | None = None
|
||||
for feature in feature_names:
|
||||
values = sorted({float(features.get(feature, 0.0)) for features, _ in rows})
|
||||
for left_value, right_value in zip(values, values[1:], strict=False):
|
||||
threshold = (left_value + right_value) / 2.0
|
||||
left_rows = [
|
||||
row for row in rows if float(row[0].get(feature, 0.0)) <= threshold
|
||||
]
|
||||
right_rows = [
|
||||
row for row in rows if float(row[0].get(feature, 0.0)) > threshold
|
||||
]
|
||||
if len(left_rows) < min_leaf or len(right_rows) < min_leaf:
|
||||
continue
|
||||
impurity = (
|
||||
len(left_rows) * _gini([label for _, label in left_rows])
|
||||
+ len(right_rows) * _gini([label for _, label in right_rows])
|
||||
) / len(rows)
|
||||
gain = base_impurity - impurity
|
||||
candidate = (gain, feature, threshold, left_rows, right_rows)
|
||||
if best is None or _better_split(candidate, best):
|
||||
best = candidate
|
||||
if best is None or best[0] <= 1e-12:
|
||||
return node
|
||||
node.update(
|
||||
{
|
||||
"feature": best[1],
|
||||
"threshold": round(best[2], 12),
|
||||
"left": _train_tree(
|
||||
best[3],
|
||||
feature_names=feature_names,
|
||||
max_depth=max_depth,
|
||||
min_leaf=min_leaf,
|
||||
depth=depth + 1,
|
||||
),
|
||||
"right": _train_tree(
|
||||
best[4],
|
||||
feature_names=feature_names,
|
||||
max_depth=max_depth,
|
||||
min_leaf=min_leaf,
|
||||
depth=depth + 1,
|
||||
),
|
||||
}
|
||||
)
|
||||
return node
|
||||
|
||||
|
||||
def _better_split(
|
||||
candidate: tuple[float, str, float, Any, Any],
|
||||
current: tuple[float, str, float, Any, Any],
|
||||
) -> bool:
|
||||
if candidate[0] > current[0] + 1e-12:
|
||||
return True
|
||||
if abs(candidate[0] - current[0]) <= 1e-12:
|
||||
return (candidate[1], candidate[2]) < (current[1], current[2])
|
||||
return False
|
||||
|
||||
|
||||
def _predict_tree(tree: dict[str, Any], features: dict[str, float]) -> str:
|
||||
node = tree
|
||||
while "feature" in node:
|
||||
feature = str(node["feature"])
|
||||
threshold = float(node["threshold"])
|
||||
node = (
|
||||
_object(node.get("left"), "E38 tree left")
|
||||
if features.get(feature, 0.0) <= threshold
|
||||
else _object(node.get("right"), "E38 tree right")
|
||||
)
|
||||
prediction = node.get("prediction")
|
||||
if not isinstance(prediction, str) or not prediction:
|
||||
raise E38PerceptionBaselineError("E38 tree prediction is invalid")
|
||||
return prediction
|
||||
|
||||
|
||||
def _dimension_metrics(
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
dimension: str,
|
||||
target: float,
|
||||
) -> dict[str, Any]:
|
||||
confusion: Counter[tuple[str, str]] = Counter()
|
||||
stratum_counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
correct = 0
|
||||
for row in rows:
|
||||
reference = str(row["reference"][dimension])
|
||||
prediction = str(row["prediction"][dimension])
|
||||
confusion[(reference, prediction)] += 1
|
||||
matched = reference == prediction
|
||||
correct += matched
|
||||
stratum_counts[str(row["source_stratum"])][
|
||||
"correct" if matched else "incorrect"
|
||||
] += 1
|
||||
total = len(rows)
|
||||
accuracy = correct / total if total else 0.0
|
||||
return {
|
||||
"correct": correct,
|
||||
"incorrect": total - correct,
|
||||
"total": total,
|
||||
"accuracy": round(accuracy, 6),
|
||||
"target": target,
|
||||
"passed": accuracy >= target,
|
||||
"confusion": [
|
||||
{
|
||||
"reference": reference,
|
||||
"prediction": prediction,
|
||||
"count": count,
|
||||
}
|
||||
for (reference, prediction), count in sorted(
|
||||
confusion.items(),
|
||||
key=lambda item: (-item[1], item[0]),
|
||||
)
|
||||
],
|
||||
"by_stratum": {
|
||||
stratum: {
|
||||
"correct": counts["correct"],
|
||||
"incorrect": counts["incorrect"],
|
||||
"total": sum(counts.values()),
|
||||
"accuracy": round(
|
||||
counts["correct"] / sum(counts.values()),
|
||||
6,
|
||||
),
|
||||
}
|
||||
for stratum, counts in sorted(stratum_counts.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _gini(labels: list[str]) -> float:
|
||||
counts = Counter(labels)
|
||||
total = len(labels)
|
||||
return 1.0 - sum((count / total) ** 2 for count in counts.values())
|
||||
|
||||
|
||||
def _span(value: object, axis: int) -> float:
|
||||
if not isinstance(value, list) or len(value) != 2:
|
||||
return -1.0
|
||||
if axis == 0 and all(isinstance(item, (int, float)) for item in value):
|
||||
return float(value[1]) - float(value[0])
|
||||
if not all(
|
||||
isinstance(item, list)
|
||||
and len(item) > axis
|
||||
and isinstance(item[axis], (int, float))
|
||||
for item in value
|
||||
):
|
||||
return -1.0
|
||||
return float(value[1][axis]) - float(value[0][axis])
|
||||
|
||||
|
||||
def _number_or(value: object, fallback: float) -> float:
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
parsed = float(value)
|
||||
if math.isfinite(parsed):
|
||||
return parsed
|
||||
return fallback
|
||||
|
||||
|
||||
def _validate_profile(profile: dict[str, Any]) -> None:
|
||||
source = _object(profile.get("source"), "E38 source")
|
||||
model = _object(profile.get("model"), "E38 model")
|
||||
dimensions = _object(model.get("dimensions"), "E38 model dimensions")
|
||||
targets = _object(profile.get("targets"), "E38 targets")
|
||||
if (
|
||||
profile.get("schema_version") != E38_PROFILE_SCHEMA
|
||||
or profile.get("profile_id")
|
||||
!= "e38-ravnoves00-r1-development-cart/v1"
|
||||
or source.get("session_id") != _SOURCE_SESSION_ID
|
||||
or source.get("display_name") != _SOURCE_DISPLAY_NAME
|
||||
or not isinstance(source.get("acceptance_result_id"), str)
|
||||
or not str(source["acceptance_result_id"]).startswith(
|
||||
"e37-ravnoves-acceptance-"
|
||||
)
|
||||
or not isinstance(source.get("materialization_id"), str)
|
||||
or not str(source["materialization_id"]).startswith("e30-materialization-")
|
||||
or model.get("type") != "deterministic-shallow-cart"
|
||||
or set(dimensions) != set(_DIMENSIONS)
|
||||
or any(
|
||||
not isinstance(dimensions[name], dict)
|
||||
or not isinstance(dimensions[name].get("include_source_time"), bool)
|
||||
or not isinstance(dimensions[name].get("max_depth"), int)
|
||||
or not 1 <= dimensions[name]["max_depth"] <= 8
|
||||
or not isinstance(dimensions[name].get("min_leaf"), int)
|
||||
or not 2 <= dimensions[name]["min_leaf"] <= 32
|
||||
for name in _DIMENSIONS
|
||||
)
|
||||
or any(targets.get(f"{name}_target") != 0.9 for name in _DIMENSIONS)
|
||||
or targets.get("accounting_target") != 1.0
|
||||
or targets.get("maximum_false_free_claims") != 0
|
||||
or profile.get("authority") != _AUTHORITY
|
||||
):
|
||||
raise E38PerceptionBaselineError("E38 profile contract changed")
|
||||
|
||||
|
||||
def _artifact(path: Path, role: str) -> dict[str, Any]:
|
||||
return {
|
||||
"role": role,
|
||||
"path": path.name,
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise E38PerceptionBaselineError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise E38PerceptionBaselineError(f"invalid JSON: {path.name}") from exc
|
||||
return _object(value, path.name)
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
with path.open("r", encoding="utf-8-sig") as stream:
|
||||
for line in stream:
|
||||
rows.append(_object(json.loads(line), path.name))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise E38PerceptionBaselineError(f"invalid JSONL: {path.name}") from exc
|
||||
return rows
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
for row in rows:
|
||||
stream.write(
|
||||
json.dumps(
|
||||
row,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
Reference in New Issue
Block a user