feat(perception): stabilize pre-capture methodology
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,930 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Development-only E40 feature and split audit.
|
||||
|
||||
The script intentionally rejects validation rows before feature extraction. It
|
||||
compares route-coordinate-free candidates under two leakage-resistant
|
||||
protocols:
|
||||
|
||||
* contiguous source-time folds;
|
||||
* whole detector tracks plus 50-frame geometry scene windows.
|
||||
|
||||
It never writes a model or a sealed result. A production E40 profile can only
|
||||
be frozen after one fixed candidate passes the development gate here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from k1link.compute.e38_perception_baseline import _predict_tree, _train_tree
|
||||
|
||||
LABELS = (
|
||||
"background-or-noise",
|
||||
"object-present",
|
||||
"occupied-environment",
|
||||
)
|
||||
TARGET = 0.9
|
||||
|
||||
CATEGORIES = {
|
||||
"stratum": ("agree", "camera-only", "conflict", "geometry-only", "unknown"),
|
||||
"range": ("near", "middle", "far", "unavailable"),
|
||||
"geometry": (
|
||||
"agree",
|
||||
"conflict",
|
||||
"single-source-camera",
|
||||
"single-source-geometry",
|
||||
"unknown",
|
||||
"unavailable",
|
||||
),
|
||||
"label": ("car", "person", "truck", "bicycle", "motorcycle", "bus", "none"),
|
||||
"reason": (
|
||||
"camera-semantic-without-qualified-occupied-lidar-support",
|
||||
"camera-semantic-with-connected-occupied-lidar-support",
|
||||
"semantic-observation-not-current",
|
||||
"camera-object-region-observed-as-local-surface",
|
||||
"none",
|
||||
),
|
||||
"association": ("vehicle", "person", "bicycle", "motorcycle", "none"),
|
||||
"motion": ("unknown", "static", "dynamic", "none"),
|
||||
"camera_motion": ("unknown", "static", "dynamic", "none"),
|
||||
"semantic_current": ("true", "false", "none"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parse_args()
|
||||
acceptance_rows = _read_jsonl(args.acceptance_root / "acceptance-items.jsonl")
|
||||
materialization_rows = _read_jsonl(args.materialization_root / "materialized-items.jsonl")
|
||||
materialization_by_id = {str(row["item_id"]): row for row in materialization_rows}
|
||||
development = [row for row in acceptance_rows if row.get("split") == "development"]
|
||||
if len(development) != 340:
|
||||
raise RuntimeError("E40 development denominator must remain 340")
|
||||
if any(row.get("split") != "development" for row in development):
|
||||
raise RuntimeError("E40 analysis received a non-development row")
|
||||
|
||||
feature_rows = [
|
||||
_features(
|
||||
acceptance=row,
|
||||
materialization=materialization_by_id[str(row["item_id"])],
|
||||
materialization_root=args.materialization_root,
|
||||
)
|
||||
for row in development
|
||||
]
|
||||
feature_names = sorted({name for row in feature_rows for name in row})
|
||||
matrix = np.asarray(
|
||||
[[row.get(name, 0.0) for name in feature_names] for row in feature_rows],
|
||||
dtype=np.float64,
|
||||
)
|
||||
labels = np.asarray(
|
||||
[LABELS.index(str(row["reference"]["presence"])) for row in development],
|
||||
dtype=np.int64,
|
||||
)
|
||||
if not np.isfinite(matrix).all():
|
||||
raise RuntimeError("E40 development features contain non-finite values")
|
||||
|
||||
assignments = {
|
||||
"contiguous-source-time-five-fold": _contiguous_folds(development),
|
||||
"whole-track-or-scene-window-five-fold": _track_scene_folds(
|
||||
development,
|
||||
materialization_by_id,
|
||||
),
|
||||
}
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"development_items": len(development),
|
||||
"feature_dimensions": len(feature_names),
|
||||
"validation_rows_loaded_for_features": 0,
|
||||
"protocols": {
|
||||
name: {str(fold): count for fold, count in sorted(Counter(values).items())}
|
||||
for name, values in assignments.items()
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
feature_sets = {
|
||||
"structured": [
|
||||
index
|
||||
for index, name in enumerate(feature_names)
|
||||
if not _source_array_feature(name) and not name.startswith("image_")
|
||||
],
|
||||
"structured-shape": [
|
||||
index for index, name in enumerate(feature_names) if not name.startswith("image_")
|
||||
],
|
||||
"structured-image": [
|
||||
index
|
||||
for index, name in enumerate(feature_names)
|
||||
if not _source_array_feature(name)
|
||||
or name.startswith("candidate_class_fraction_")
|
||||
or name.startswith("image_")
|
||||
],
|
||||
"all": list(range(len(feature_names))),
|
||||
}
|
||||
candidates = [
|
||||
("structured-knn-k3", "knn", 3, "structured"),
|
||||
("structured-softmax-l2-0.01", "softmax", 0.01, "structured"),
|
||||
(
|
||||
"structured-shape-softmax-l2-0.01",
|
||||
"softmax",
|
||||
0.01,
|
||||
"structured-shape",
|
||||
),
|
||||
(
|
||||
"structured-image-softmax-l2-0.01",
|
||||
"softmax",
|
||||
0.01,
|
||||
"structured-image",
|
||||
),
|
||||
("all-softmax-l2-0.01", "softmax", 0.01, "all"),
|
||||
("all-softmax-l2-0.03", "softmax", 0.03, "all"),
|
||||
("all-softmax-l2-0.1", "softmax", 0.1, "all"),
|
||||
(
|
||||
"hierarchical-structured-softmax-l2-0.01",
|
||||
"hierarchical-softmax",
|
||||
0.01,
|
||||
"structured",
|
||||
),
|
||||
(
|
||||
"hierarchical-structured-image-softmax-l2-0.01",
|
||||
"hierarchical-softmax",
|
||||
0.01,
|
||||
"structured-image",
|
||||
),
|
||||
(
|
||||
"hierarchical-all-softmax-l2-0.01",
|
||||
"hierarchical-softmax",
|
||||
0.01,
|
||||
"all",
|
||||
),
|
||||
]
|
||||
candidate_results: list[dict[str, Any]] = []
|
||||
for candidate_name, model_type, model_parameter, feature_set in candidates:
|
||||
columns = feature_sets[feature_set]
|
||||
result = {
|
||||
protocol: _cross_validate(
|
||||
matrix=matrix[:, columns],
|
||||
labels=labels,
|
||||
feature_names=[feature_names[index] for index in columns],
|
||||
feature_rows=feature_rows,
|
||||
source_strata=[str(row["source_stratum"]) for row in development],
|
||||
assignments=folds,
|
||||
model_type=model_type,
|
||||
model_parameter=model_parameter,
|
||||
)
|
||||
for protocol, folds in assignments.items()
|
||||
}
|
||||
candidate_results.append(
|
||||
{
|
||||
"candidate": candidate_name,
|
||||
"feature_set": feature_set,
|
||||
"feature_dimensions": len(columns),
|
||||
"accuracy": {protocol: row["accuracy"] for protocol, row in result.items()},
|
||||
"by_stratum": {protocol: row["by_stratum"] for protocol, row in result.items()},
|
||||
"by_fold": {protocol: row["by_fold"] for protocol, row in result.items()},
|
||||
"passed_both": all(row["accuracy"] >= TARGET for row in result.values()),
|
||||
}
|
||||
)
|
||||
print(json.dumps(candidate_results, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _features(
|
||||
*,
|
||||
acceptance: dict[str, Any],
|
||||
materialization: dict[str, Any],
|
||||
materialization_root: Path,
|
||||
) -> dict[str, float]:
|
||||
snapshot = _object(materialization.get("e29_snapshot"))
|
||||
evidence = _object(materialization.get("materialization"))
|
||||
values: dict[str, float] = {}
|
||||
observed = {
|
||||
"stratum": materialization.get("stratum"),
|
||||
"range": materialization.get("range_bucket"),
|
||||
"geometry": snapshot.get("geometry_status"),
|
||||
"label": snapshot.get("label") or "none",
|
||||
"reason": snapshot.get("geometry_reason") or "none",
|
||||
"association": snapshot.get("association_group") or "none",
|
||||
"motion": snapshot.get("motion_state") or "none",
|
||||
"camera_motion": snapshot.get("camera_motion_state") or "none",
|
||||
"semantic_current": (
|
||||
str(snapshot.get("semantic_current")).lower()
|
||||
if snapshot.get("semantic_current") is not None
|
||||
else "none"
|
||||
),
|
||||
}
|
||||
for prefix, categories in CATEGORIES.items():
|
||||
for category in categories:
|
||||
values[f"{prefix}={category}"] = float(observed[prefix] == category)
|
||||
|
||||
for name in (
|
||||
"selected_point_count",
|
||||
"candidate_point_count",
|
||||
"rejected_candidate_point_count",
|
||||
"projected_point_count",
|
||||
"frame_point_count",
|
||||
):
|
||||
values[name] = math.log1p(max(0.0, _number(evidence.get(name), 0.0)))
|
||||
values["detector_score"] = _number(evidence.get("detector_score"), -1.0)
|
||||
for name in (
|
||||
"nearest_range_m",
|
||||
"point_count",
|
||||
"voxel_count",
|
||||
"score",
|
||||
"camera_motion_confidence",
|
||||
"range_m",
|
||||
):
|
||||
raw = _number(snapshot.get(name), -1.0)
|
||||
values[name] = (
|
||||
math.log1p(raw) if name in {"point_count", "voxel_count"} and raw >= 0.0 else raw
|
||||
)
|
||||
|
||||
bbox = snapshot.get("bbox_xyxy")
|
||||
if isinstance(bbox, list) and len(bbox) == 4:
|
||||
x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox)
|
||||
width = max(0.0, x2 - x1) / 800.0
|
||||
height = max(0.0, y2 - y1) / 600.0
|
||||
values.update(
|
||||
{
|
||||
"bbox_present": 1.0,
|
||||
"bbox_center_x": (x1 + x2) / 1600.0,
|
||||
"bbox_center_y": (y1 + y2) / 1200.0,
|
||||
"bbox_width": width,
|
||||
"bbox_height": height,
|
||||
"bbox_area": width * height,
|
||||
"bbox_aspect": width / (height + 1e-6),
|
||||
}
|
||||
)
|
||||
else:
|
||||
values.update(
|
||||
{
|
||||
"bbox_present": 0.0,
|
||||
"bbox_center_x": -1.0,
|
||||
"bbox_center_y": -1.0,
|
||||
"bbox_width": -1.0,
|
||||
"bbox_height": -1.0,
|
||||
"bbox_area": -1.0,
|
||||
"bbox_aspect": -1.0,
|
||||
}
|
||||
)
|
||||
|
||||
support = snapshot.get("support")
|
||||
support = support if isinstance(support, dict) else {}
|
||||
support_names = (
|
||||
"below_surface_points_in_bbox",
|
||||
"classified_points_in_bbox",
|
||||
"connected_occupied_points",
|
||||
"connected_occupied_voxels",
|
||||
"occupied_points_in_bbox",
|
||||
"projected_points_in_bbox",
|
||||
"surface_points_in_bbox",
|
||||
)
|
||||
support_values = {name: _number(support.get(name), 0.0) for name in support_names}
|
||||
values.update(
|
||||
{f"support_{name}": math.log1p(max(0.0, value)) for name, value in support_values.items()}
|
||||
)
|
||||
projected = max(1.0, support_values["projected_points_in_bbox"])
|
||||
occupied = max(1.0, support_values["occupied_points_in_bbox"])
|
||||
values.update(
|
||||
{
|
||||
"support_occupied_fraction": (support_values["occupied_points_in_bbox"] / projected),
|
||||
"support_classified_fraction": (
|
||||
support_values["classified_points_in_bbox"] / projected
|
||||
),
|
||||
"support_surface_fraction": (support_values["surface_points_in_bbox"] / projected),
|
||||
"support_connected_fraction": (support_values["connected_occupied_points"] / occupied),
|
||||
}
|
||||
)
|
||||
_span_features(values, "bounds", snapshot.get("bounds_map_xyz_m"))
|
||||
_range_span(values, "height_span", snapshot.get("height_range_m"))
|
||||
_range_span(
|
||||
values,
|
||||
"occupied_height_span",
|
||||
snapshot.get("occupied_height_range_m"),
|
||||
)
|
||||
|
||||
artifact = _object(materialization.get("artifact"))
|
||||
artifact_path = materialization_root / str(artifact.get("path"))
|
||||
with np.load(artifact_path, allow_pickle=False) as arrays:
|
||||
pixels = arrays["projected_pixels_xy"]
|
||||
candidate_mask = arrays["projected_candidate_mask"].astype(bool)
|
||||
selected_mask = arrays["projected_selected_mask"].astype(bool)
|
||||
sensor_position = arrays["sensor_position_map_xyz_m"]
|
||||
sensor_orientation = arrays["sensor_orientation_map_from_lidar_xyzw"]
|
||||
_named_point_statistics(
|
||||
values,
|
||||
"candidate_lidar",
|
||||
arrays["candidate_points_map_xyz_m"],
|
||||
sensor_position,
|
||||
sensor_orientation,
|
||||
)
|
||||
_named_point_statistics(
|
||||
values,
|
||||
"selected_lidar",
|
||||
arrays["selected_points_map_xyz_m"],
|
||||
sensor_position,
|
||||
sensor_orientation,
|
||||
)
|
||||
projection_width = max(1, int(evidence.get("projection_width", 800)))
|
||||
projection_height = max(1, int(evidence.get("projection_height", 600)))
|
||||
_named_pixel_statistics(
|
||||
values,
|
||||
"candidate_pixel",
|
||||
pixels[candidate_mask],
|
||||
projection_width,
|
||||
projection_height,
|
||||
)
|
||||
_named_pixel_statistics(
|
||||
values,
|
||||
"selected_pixel",
|
||||
pixels[selected_mask],
|
||||
projection_width,
|
||||
projection_height,
|
||||
)
|
||||
_named_quantiles(
|
||||
values,
|
||||
"candidate_depth",
|
||||
arrays["projected_depth_m"][candidate_mask],
|
||||
)
|
||||
_named_quantiles(
|
||||
values,
|
||||
"selected_depth",
|
||||
arrays["projected_depth_m"][selected_mask],
|
||||
)
|
||||
_named_quantiles(
|
||||
values,
|
||||
"candidate_height",
|
||||
arrays["projected_point_height_m"][candidate_mask],
|
||||
)
|
||||
_named_quantiles(
|
||||
values,
|
||||
"selected_height",
|
||||
arrays["projected_point_height_m"][selected_mask],
|
||||
)
|
||||
point_classes = arrays["projected_point_class"][candidate_mask]
|
||||
for index in range(8):
|
||||
values[f"candidate_class_fraction_{index}"] = (
|
||||
float(np.mean(point_classes == index)) if point_classes.size else 0.0
|
||||
)
|
||||
_image_features(
|
||||
values,
|
||||
materialization=materialization,
|
||||
materialization_root=materialization_root,
|
||||
candidate_pixels=pixels[candidate_mask],
|
||||
projection_width=projection_width,
|
||||
projection_height=projection_height,
|
||||
)
|
||||
# Source frame, session time, review ordinal, track ID and absolute map
|
||||
# coordinates are deliberately absent.
|
||||
return values
|
||||
|
||||
|
||||
def _cross_validate(
|
||||
*,
|
||||
matrix: np.ndarray,
|
||||
labels: np.ndarray,
|
||||
feature_names: list[str],
|
||||
feature_rows: list[dict[str, float]],
|
||||
source_strata: list[str],
|
||||
assignments: np.ndarray,
|
||||
model_type: str,
|
||||
model_parameter: object,
|
||||
) -> dict[str, Any]:
|
||||
predictions = np.full(len(labels), -1, dtype=np.int64)
|
||||
for fold in range(5):
|
||||
train_indices = np.flatnonzero(assignments != fold)
|
||||
test_indices = np.flatnonzero(assignments == fold)
|
||||
train, test = _robust_transform(
|
||||
matrix[train_indices],
|
||||
matrix[test_indices],
|
||||
)
|
||||
if model_type == "knn":
|
||||
distances = np.mean(
|
||||
np.square(test[:, None, :] - train[None, :, :]),
|
||||
axis=2,
|
||||
)
|
||||
nearest = np.argsort(distances, axis=1, kind="stable")[:, : int(model_parameter)]
|
||||
for local_index, neighbor_indices in enumerate(nearest):
|
||||
votes = Counter(labels[train_indices][neighbor_indices])
|
||||
predictions[test_indices[local_index]] = sorted(
|
||||
votes.items(),
|
||||
key=lambda item: (-item[1], item[0]),
|
||||
)[0][0]
|
||||
elif model_type == "softmax":
|
||||
weights = _train_softmax(
|
||||
train,
|
||||
labels[train_indices],
|
||||
l2=float(model_parameter),
|
||||
)
|
||||
predictions[test_indices] = np.argmax(
|
||||
np.column_stack((test, np.ones(len(test)))) @ weights,
|
||||
axis=1,
|
||||
)
|
||||
elif model_type == "hierarchical-softmax":
|
||||
_predict_hierarchical_fold(
|
||||
predictions=predictions,
|
||||
train_indices=train_indices,
|
||||
test_indices=test_indices,
|
||||
training=train,
|
||||
testing=test,
|
||||
labels=labels,
|
||||
source_strata=source_strata,
|
||||
l2=float(model_parameter),
|
||||
)
|
||||
elif model_type == "tree":
|
||||
depth, min_leaf = model_parameter # type: ignore[misc]
|
||||
tree = _train_tree(
|
||||
[
|
||||
(
|
||||
{name: feature_rows[index].get(name, 0.0) for name in feature_names},
|
||||
LABELS[int(labels[index])],
|
||||
)
|
||||
for index in train_indices
|
||||
],
|
||||
feature_names=feature_names,
|
||||
max_depth=int(depth),
|
||||
min_leaf=int(min_leaf),
|
||||
)
|
||||
for index in test_indices:
|
||||
predictions[index] = LABELS.index(_predict_tree(tree, feature_rows[int(index)]))
|
||||
else:
|
||||
raise RuntimeError(f"unsupported model: {model_type}")
|
||||
accuracy = float(np.mean(predictions == labels))
|
||||
stratum_metrics = {}
|
||||
for stratum in sorted(set(source_strata)):
|
||||
indices = np.asarray(
|
||||
[index for index, value in enumerate(source_strata) if value == stratum],
|
||||
dtype=np.int64,
|
||||
)
|
||||
stratum_metrics[stratum] = round(
|
||||
float(np.mean(predictions[indices] == labels[indices])),
|
||||
6,
|
||||
)
|
||||
fold_metrics = {}
|
||||
for fold in range(5):
|
||||
indices = np.flatnonzero(assignments == fold)
|
||||
fold_metrics[str(fold)] = round(
|
||||
float(np.mean(predictions[indices] == labels[indices])),
|
||||
6,
|
||||
)
|
||||
return {
|
||||
"accuracy": round(accuracy, 6),
|
||||
"correct": int(np.sum(predictions == labels)),
|
||||
"incorrect": int(np.sum(predictions != labels)),
|
||||
"passed": accuracy >= TARGET,
|
||||
"by_stratum": stratum_metrics,
|
||||
"by_fold": fold_metrics,
|
||||
"confusion": [
|
||||
{
|
||||
"reference": LABELS[reference],
|
||||
"prediction": LABELS[prediction],
|
||||
"count": count,
|
||||
}
|
||||
for (reference, prediction), count in sorted(
|
||||
Counter(
|
||||
zip(
|
||||
labels.tolist(),
|
||||
predictions.tolist(),
|
||||
strict=True,
|
||||
)
|
||||
).items(),
|
||||
key=lambda item: (-item[1], item[0]),
|
||||
)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _predict_hierarchical_fold(
|
||||
*,
|
||||
predictions: np.ndarray,
|
||||
train_indices: np.ndarray,
|
||||
test_indices: np.ndarray,
|
||||
training: np.ndarray,
|
||||
testing: np.ndarray,
|
||||
labels: np.ndarray,
|
||||
source_strata: list[str],
|
||||
l2: float,
|
||||
) -> None:
|
||||
fixed = {
|
||||
"conflict": LABELS.index("background-or-noise"),
|
||||
"agree": LABELS.index("object-present"),
|
||||
"unknown": LABELS.index("object-present"),
|
||||
# A geometry-only cluster is positive occupied evidence, but without
|
||||
# camera semantics it must not be promoted to a named object. Keeping
|
||||
# it occupied is the conservative product state.
|
||||
"geometry-only": LABELS.index("occupied-environment"),
|
||||
}
|
||||
for source_index in test_indices:
|
||||
stratum = source_strata[int(source_index)]
|
||||
if stratum in fixed:
|
||||
predictions[source_index] = fixed[stratum]
|
||||
for stratum, fallback in (("camera-only", LABELS.index("object-present")),):
|
||||
local_training = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, source_index in enumerate(train_indices)
|
||||
if source_strata[int(source_index)] == stratum
|
||||
],
|
||||
dtype=np.int64,
|
||||
)
|
||||
local_testing = np.asarray(
|
||||
[
|
||||
index
|
||||
for index, source_index in enumerate(test_indices)
|
||||
if source_strata[int(source_index)] == stratum
|
||||
],
|
||||
dtype=np.int64,
|
||||
)
|
||||
if not len(local_testing):
|
||||
continue
|
||||
observed = sorted(set(labels[train_indices][local_training].tolist()))
|
||||
if len(local_training) < 10 or len(observed) < 2:
|
||||
predictions[test_indices[local_testing]] = fallback
|
||||
continue
|
||||
weights = _train_softmax(
|
||||
training[local_training],
|
||||
labels[train_indices][local_training],
|
||||
l2=l2,
|
||||
)
|
||||
predictions[test_indices[local_testing]] = np.argmax(
|
||||
np.column_stack(
|
||||
(
|
||||
testing[local_testing],
|
||||
np.ones(len(local_testing)),
|
||||
)
|
||||
)
|
||||
@ weights,
|
||||
axis=1,
|
||||
)
|
||||
|
||||
|
||||
def _train_softmax(
|
||||
matrix: np.ndarray,
|
||||
labels: np.ndarray,
|
||||
*,
|
||||
l2: float,
|
||||
) -> np.ndarray:
|
||||
rows, dimensions = matrix.shape
|
||||
design = np.column_stack((matrix, np.ones(rows)))
|
||||
targets = np.eye(len(LABELS), dtype=np.float64)[labels]
|
||||
weights = np.zeros((dimensions + 1, len(LABELS)), dtype=np.float64)
|
||||
first_moment = np.zeros_like(weights)
|
||||
second_moment = np.zeros_like(weights)
|
||||
for step in range(1, 1201):
|
||||
logits = design @ weights
|
||||
logits -= np.max(logits, axis=1, keepdims=True)
|
||||
probabilities = np.exp(logits)
|
||||
probabilities /= np.sum(probabilities, axis=1, keepdims=True)
|
||||
regularizer = np.vstack((weights[:-1], np.zeros((1, len(LABELS)))))
|
||||
gradient = design.T @ (probabilities - targets) / rows
|
||||
gradient += l2 * regularizer
|
||||
first_moment = 0.9 * first_moment + 0.1 * gradient
|
||||
second_moment = 0.999 * second_moment + 0.001 * np.square(gradient)
|
||||
corrected_first = first_moment / (1.0 - 0.9**step)
|
||||
corrected_second = second_moment / (1.0 - 0.999**step)
|
||||
weights -= 0.03 * corrected_first / (np.sqrt(corrected_second) + 1e-8)
|
||||
return weights
|
||||
|
||||
|
||||
def _robust_transform(
|
||||
training: np.ndarray,
|
||||
testing: np.ndarray,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
median = np.median(training, axis=0)
|
||||
scale = np.percentile(training, 75, axis=0) - np.percentile(
|
||||
training,
|
||||
25,
|
||||
axis=0,
|
||||
)
|
||||
scale[scale < 1e-8] = 1.0
|
||||
return (
|
||||
np.clip((training - median) / scale, -10.0, 10.0),
|
||||
np.clip((testing - median) / scale, -10.0, 10.0),
|
||||
)
|
||||
|
||||
|
||||
def _source_array_feature(name: str) -> bool:
|
||||
return name.startswith(
|
||||
(
|
||||
"candidate_lidar_",
|
||||
"selected_lidar_",
|
||||
"candidate_pixel_",
|
||||
"selected_pixel_",
|
||||
"candidate_depth_",
|
||||
"selected_depth_",
|
||||
"candidate_height_",
|
||||
"selected_height_",
|
||||
"candidate_class_fraction_",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _contiguous_folds(rows: list[dict[str, Any]]) -> np.ndarray:
|
||||
order = np.argsort(
|
||||
[int(row["source_frame_index"]) for row in rows],
|
||||
kind="stable",
|
||||
)
|
||||
assignments = np.empty(len(rows), dtype=np.int64)
|
||||
for fold, indices in enumerate(np.array_split(order, 5)):
|
||||
assignments[indices] = fold
|
||||
return assignments
|
||||
|
||||
|
||||
def _track_scene_folds(
|
||||
rows: list[dict[str, Any]],
|
||||
materialization_by_id: dict[str, dict[str, Any]],
|
||||
) -> np.ndarray:
|
||||
assignments: list[int] = []
|
||||
for row in rows:
|
||||
snapshot = _object(materialization_by_id[str(row["item_id"])].get("e29_snapshot"))
|
||||
track_id = snapshot.get("track_id")
|
||||
group = (
|
||||
f"track:{track_id}"
|
||||
if track_id is not None
|
||||
else f"scene:{int(row['source_frame_index']) // 50}"
|
||||
)
|
||||
digest = hashlib.sha256(f"e40:{group}".encode()).hexdigest()
|
||||
assignments.append(int(digest[:8], 16) % 5)
|
||||
return np.asarray(assignments, dtype=np.int64)
|
||||
|
||||
|
||||
def _named_point_statistics(
|
||||
values: dict[str, float],
|
||||
prefix: str,
|
||||
points_map: np.ndarray,
|
||||
sensor_position: np.ndarray,
|
||||
sensor_orientation_xyzw: np.ndarray,
|
||||
) -> None:
|
||||
points = np.asarray(points_map, dtype=np.float64)
|
||||
if points.ndim != 2 or points.shape[1] != 3 or not len(points):
|
||||
for axis in "xyz":
|
||||
_named_quantiles(values, f"{prefix}_{axis}", np.asarray([]))
|
||||
for index in range(3):
|
||||
values[f"{prefix}_covariance_ratio_{index}"] = 0.0
|
||||
return
|
||||
rotation = _rotation_matrix(sensor_orientation_xyzw)
|
||||
points_local = (points - np.asarray(sensor_position)) @ rotation
|
||||
for axis, index in zip("xyz", range(3), strict=True):
|
||||
_named_quantiles(values, f"{prefix}_{axis}", points_local[:, index])
|
||||
if len(points_local) >= 3:
|
||||
eigenvalues = np.maximum(
|
||||
np.linalg.eigvalsh(np.cov(points_local, rowvar=False)),
|
||||
0.0,
|
||||
)
|
||||
else:
|
||||
eigenvalues = np.zeros(3)
|
||||
ratios = eigenvalues / (float(np.sum(eigenvalues)) + 1e-9)
|
||||
for index, ratio in enumerate(ratios):
|
||||
values[f"{prefix}_covariance_ratio_{index}"] = float(ratio)
|
||||
|
||||
|
||||
def _rotation_matrix(quaternion_xyzw: np.ndarray) -> np.ndarray:
|
||||
x, y, z, w = (float(value) for value in quaternion_xyzw)
|
||||
return np.asarray(
|
||||
[
|
||||
[
|
||||
1.0 - 2.0 * (y * y + z * z),
|
||||
2.0 * (x * y - z * w),
|
||||
2.0 * (x * z + y * w),
|
||||
],
|
||||
[
|
||||
2.0 * (x * y + z * w),
|
||||
1.0 - 2.0 * (x * x + z * z),
|
||||
2.0 * (y * z - x * w),
|
||||
],
|
||||
[
|
||||
2.0 * (x * z - y * w),
|
||||
2.0 * (y * z + x * w),
|
||||
1.0 - 2.0 * (x * x + y * y),
|
||||
],
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
|
||||
|
||||
def _named_pixel_statistics(
|
||||
values: dict[str, float],
|
||||
prefix: str,
|
||||
pixels: np.ndarray,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> None:
|
||||
points = np.asarray(pixels, dtype=np.float64)
|
||||
if points.ndim != 2 or points.shape[1] != 2 or not len(points):
|
||||
_named_quantiles(values, f"{prefix}_x", np.asarray([]))
|
||||
_named_quantiles(values, f"{prefix}_y", np.asarray([]))
|
||||
values[f"{prefix}_span_x"] = -1.0
|
||||
values[f"{prefix}_span_y"] = -1.0
|
||||
values[f"{prefix}_density"] = -1.0
|
||||
return
|
||||
x = points[:, 0] / width
|
||||
y = points[:, 1] / height
|
||||
_named_quantiles(values, f"{prefix}_x", x)
|
||||
_named_quantiles(values, f"{prefix}_y", y)
|
||||
span_x = max(float(np.max(x) - np.min(x)), 1.0 / width)
|
||||
span_y = max(float(np.max(y) - np.min(y)), 1.0 / height)
|
||||
values[f"{prefix}_span_x"] = span_x
|
||||
values[f"{prefix}_span_y"] = span_y
|
||||
values[f"{prefix}_density"] = len(points) / (span_x * span_y * width * height + 1.0)
|
||||
|
||||
|
||||
def _image_features(
|
||||
values: dict[str, float],
|
||||
*,
|
||||
materialization: dict[str, Any],
|
||||
materialization_root: Path,
|
||||
candidate_pixels: np.ndarray,
|
||||
projection_width: int,
|
||||
projection_height: int,
|
||||
) -> None:
|
||||
snapshot = _object(materialization.get("e29_snapshot"))
|
||||
bbox = snapshot.get("bbox_xyxy")
|
||||
if not isinstance(bbox, list) or len(bbox) != 4:
|
||||
points = np.asarray(candidate_pixels, dtype=np.float64)
|
||||
if points.ndim != 2 or points.shape[1] != 2 or not len(points):
|
||||
_empty_image_features(values)
|
||||
return
|
||||
low = np.min(points, axis=0)
|
||||
high = np.max(points, axis=0)
|
||||
center = (low + high) / 2.0
|
||||
support = np.maximum(high - low, 48.0)
|
||||
bbox = [
|
||||
center[0] - support[0],
|
||||
center[1] - support[1],
|
||||
center[0] + support[0],
|
||||
center[1] + support[1],
|
||||
]
|
||||
x1, y1, x2, y2 = (_number(value, 0.0) for value in bbox)
|
||||
box = (
|
||||
max(0, int(x1)),
|
||||
max(0, int(y1)),
|
||||
min(projection_width, int(math.ceil(x2))),
|
||||
min(projection_height, int(math.ceil(y2))),
|
||||
)
|
||||
if box[2] <= box[0] or box[3] <= box[1]:
|
||||
_empty_image_features(values)
|
||||
return
|
||||
frame = _object(materialization.get("camera_frame"))
|
||||
with Image.open(materialization_root / str(frame.get("path"))) as source:
|
||||
rgb = (
|
||||
np.asarray(
|
||||
source.convert("RGB").crop(box).resize((32, 32), Image.Resampling.BILINEAR),
|
||||
dtype=np.float64,
|
||||
)
|
||||
/ 255.0
|
||||
)
|
||||
for channel, name in enumerate(("red", "green", "blue")):
|
||||
histogram, _ = np.histogram(
|
||||
rgb[:, :, channel],
|
||||
bins=4,
|
||||
range=(0.0, 1.0),
|
||||
)
|
||||
histogram = histogram / max(1, int(np.sum(histogram)))
|
||||
for index, fraction in enumerate(histogram):
|
||||
values[f"image_{name}_histogram_{index}"] = float(fraction)
|
||||
luma = np.mean(rgb, axis=2)
|
||||
saturation = np.max(rgb, axis=2) - np.min(rgb, axis=2)
|
||||
edge = np.concatenate(
|
||||
(
|
||||
np.abs(np.diff(luma, axis=1)).reshape(-1),
|
||||
np.abs(np.diff(luma, axis=0)).reshape(-1),
|
||||
)
|
||||
)
|
||||
values["image_luma_mean"] = float(np.mean(luma))
|
||||
values["image_luma_std"] = float(np.std(luma))
|
||||
for name, quantile in zip(
|
||||
("p10", "p25", "p50", "p75", "p90"),
|
||||
np.quantile(luma, (0.1, 0.25, 0.5, 0.75, 0.9)),
|
||||
strict=True,
|
||||
):
|
||||
values[f"image_luma_{name}"] = float(quantile)
|
||||
values["image_saturation_mean"] = float(np.mean(saturation))
|
||||
values["image_saturation_std"] = float(np.std(saturation))
|
||||
values["image_edge_mean"] = float(np.mean(edge))
|
||||
values["image_edge_p90"] = float(np.quantile(edge, 0.9))
|
||||
small_luma = (
|
||||
np.asarray(
|
||||
Image.fromarray(np.uint8(np.clip(luma * 255.0, 0.0, 255.0))).resize(
|
||||
(4, 4),
|
||||
Image.Resampling.BILINEAR,
|
||||
),
|
||||
dtype=np.float64,
|
||||
)
|
||||
/ 255.0
|
||||
)
|
||||
small_luma -= float(np.mean(small_luma))
|
||||
for y in range(4):
|
||||
for x in range(4):
|
||||
values[f"image_luma_centered_{y}_{x}"] = float(small_luma[y, x])
|
||||
|
||||
|
||||
def _empty_image_features(values: dict[str, float]) -> None:
|
||||
for channel in ("red", "green", "blue"):
|
||||
for index in range(4):
|
||||
values[f"image_{channel}_histogram_{index}"] = 0.0
|
||||
for name in (
|
||||
"mean",
|
||||
"std",
|
||||
"p10",
|
||||
"p25",
|
||||
"p50",
|
||||
"p75",
|
||||
"p90",
|
||||
):
|
||||
values[f"image_luma_{name}"] = 0.0
|
||||
values["image_saturation_mean"] = 0.0
|
||||
values["image_saturation_std"] = 0.0
|
||||
values["image_edge_mean"] = 0.0
|
||||
values["image_edge_p90"] = 0.0
|
||||
for y in range(4):
|
||||
for x in range(4):
|
||||
values[f"image_luma_centered_{y}_{x}"] = 0.0
|
||||
|
||||
|
||||
def _named_quantiles(
|
||||
values: dict[str, float],
|
||||
prefix: str,
|
||||
raw: object,
|
||||
) -> None:
|
||||
array = np.asarray(raw, dtype=np.float64).reshape(-1)
|
||||
finite = array[np.isfinite(array)]
|
||||
names = ("min", "p10", "p25", "p50", "p75", "p90", "max")
|
||||
quantiles = (
|
||||
np.quantile(finite, (0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0))
|
||||
if finite.size
|
||||
else np.full(7, -1.0)
|
||||
)
|
||||
for name, value in zip(names, quantiles, strict=True):
|
||||
values[f"{prefix}_{name}"] = float(value)
|
||||
|
||||
|
||||
def _span_features(
|
||||
values: dict[str, float],
|
||||
prefix: str,
|
||||
raw: object,
|
||||
) -> None:
|
||||
if (
|
||||
isinstance(raw, list)
|
||||
and len(raw) == 2
|
||||
and all(isinstance(item, list) and len(item) == 3 for item in raw)
|
||||
):
|
||||
for index, axis in enumerate("xyz"):
|
||||
values[f"{prefix}_{axis}_span"] = _number(
|
||||
raw[1][index],
|
||||
-1.0,
|
||||
) - _number(raw[0][index], -1.0)
|
||||
else:
|
||||
for axis in "xyz":
|
||||
values[f"{prefix}_{axis}_span"] = -1.0
|
||||
|
||||
|
||||
def _range_span(
|
||||
values: dict[str, float],
|
||||
name: str,
|
||||
raw: object,
|
||||
) -> None:
|
||||
values[name] = (
|
||||
_number(raw[1], -1.0) - _number(raw[0], -1.0)
|
||||
if isinstance(raw, list) and len(raw) == 2
|
||||
else -1.0
|
||||
)
|
||||
|
||||
|
||||
def _number(value: object, fallback: float) -> float:
|
||||
try:
|
||||
parsed = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError):
|
||||
return fallback
|
||||
return parsed if math.isfinite(parsed) else fallback
|
||||
|
||||
|
||||
def _object(value: object) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise RuntimeError("expected object")
|
||||
return value
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8-sig") as stream:
|
||||
return [_object(json.loads(line)) for line in stream]
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--acceptance-root", type=Path, required=True)
|
||||
parser.add_argument("--materialization-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
},
|
||||
"model": {
|
||||
"cross_validation_folds": 5,
|
||||
"cross_validation_seed": "e40-grouped-dev-cv",
|
||||
"feature_set": "route-coordinate-free-structured-image/v1",
|
||||
"fixed_presence_by_stratum": {
|
||||
"agree": "object-present",
|
||||
"conflict": "background-or-noise",
|
||||
"geometry-only": "occupied-environment",
|
||||
"unknown": "object-present"
|
||||
},
|
||||
"l2": 0.01,
|
||||
"learning_rate": 0.03,
|
||||
"robust_clip": 10.0,
|
||||
"steps": 1200,
|
||||
"type": "hierarchical-stratum-softmax"
|
||||
},
|
||||
"profile_id": "e40-ravnoves00-leakage-resistant-product-gate/v1",
|
||||
"schema_version": "missioncore.e40-perception-product-gate-profile/v1",
|
||||
"source": {
|
||||
"acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344",
|
||||
"display_name": "RAVNOVES00",
|
||||
"materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a",
|
||||
"session_id": "20260720T065719Z_viewer_live"
|
||||
},
|
||||
"targets": {
|
||||
"accounting_target": 1.0,
|
||||
"freshness_target": 0.9,
|
||||
"geometry_association_target": 0.9,
|
||||
"maximum_false_free_claims": 0,
|
||||
"maximum_high_severity_failures": 0,
|
||||
"presence_target": 0.9
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
},
|
||||
"policy": {
|
||||
"current_validation_semantics": "historical-evaluated-visible-validation",
|
||||
"forbidden_feature_tokens": [
|
||||
"item_id",
|
||||
"map_xyz",
|
||||
"path",
|
||||
"review_ordinal",
|
||||
"session_id",
|
||||
"session_seconds",
|
||||
"source_frame",
|
||||
"track_id"
|
||||
],
|
||||
"independent_truth_required_for_blind": true,
|
||||
"predictor_truth_separation_required": true,
|
||||
"time_block_frames": 50
|
||||
},
|
||||
"profile_id": "e41-ravnoves00-methodology-audit/v1",
|
||||
"schema_version": "missioncore.e41-methodology-audit-profile/v1",
|
||||
"source": {
|
||||
"acceptance_result_id": "e37-ravnoves-acceptance-01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344",
|
||||
"e40_package_id": "e40-worker-package-0b7c1aa6d1d31172206b125002928f9adfd8ea3c0a8f95c6caae431bfafff247",
|
||||
"e40_result_id": "e40-perception-product-gate-e96eec9fd68c3ffaaee898d46285dd329191267200011680f084c75095b92e9a",
|
||||
"materialization_id": "e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"acceptance_contract": {
|
||||
"accounting_target": 1.0,
|
||||
"freshness_target": 0.9,
|
||||
"geometry_association_target": 0.9,
|
||||
"maximum_false_free_claims": 0,
|
||||
"maximum_high_severity_failures": 0,
|
||||
"presence_target": 0.9
|
||||
},
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"navigation_or_safety_accepted": false
|
||||
},
|
||||
"blind_truth_contract": {
|
||||
"blind_fraction": 0.3,
|
||||
"engineering_acceptance_labels_are_truth": false,
|
||||
"independent_human_reviewers": 2,
|
||||
"labels_revealed_after_frozen_prediction": true,
|
||||
"partition_strategy": "connected-scene-track-time-components/v1",
|
||||
"seed": "mission-core-e43-same-k1-new-route-v1"
|
||||
},
|
||||
"capture_contract": {
|
||||
"device_model": "XGRIDS/LixelKity-K1",
|
||||
"maximum_duration_seconds": 900,
|
||||
"minimum_duration_seconds": 480,
|
||||
"required_segments": [
|
||||
{
|
||||
"kind": "control-bridge",
|
||||
"minimum_duration_seconds": 60
|
||||
},
|
||||
{
|
||||
"kind": "new-route",
|
||||
"minimum_duration_seconds": 360
|
||||
}
|
||||
],
|
||||
"required_streams": [
|
||||
"sensor.camera.right",
|
||||
"sensor.lidar.registered-map-increment",
|
||||
"sensor.pose",
|
||||
"telemetry.pipeline"
|
||||
],
|
||||
"route_policy": "control-bridge-then-new-route/v1",
|
||||
"same_device_mount_calibration_firmware_required": true
|
||||
},
|
||||
"profile_id": "e43-same-k1-new-route-truth-island/v1",
|
||||
"schema_version": "missioncore.e43-future-capture-profile/v1"
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build an immutable camera/LiDAR E40 package for Worker 006."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.compute.e40_perception_product_gate import (
|
||||
_FEATURE_CACHE_ARRAYS,
|
||||
_FEATURE_CACHE_MANIFEST,
|
||||
_FEATURE_CACHE_SCHEMA,
|
||||
E40_PACKAGE_SCHEMA,
|
||||
E40_PROFILE_SCHEMA,
|
||||
_feature_names,
|
||||
_feature_vector,
|
||||
)
|
||||
|
||||
_RUNTIME_FILES = {
|
||||
"runtime/k1link/__init__.py": "src/k1link/__init__.py",
|
||||
"runtime/k1link/compute/__init__.py": None,
|
||||
"runtime/k1link/compute/e37_acceptance_contract.py": (
|
||||
"src/k1link/compute/e37_acceptance_contract.py"
|
||||
),
|
||||
"runtime/k1link/compute/e40_perception_product_gate.py": (
|
||||
"src/k1link/compute/e40_perception_product_gate.py"
|
||||
),
|
||||
"runtime/run_e40_perception_product_gate.py": (
|
||||
"experiments/perception/worker/run_e40_perception_product_gate.py"
|
||||
),
|
||||
"runtime/validate_e40_worker_package.py": (
|
||||
"experiments/perception/worker/validate_e40_worker_package.py"
|
||||
),
|
||||
"runtime/Invoke-E40PerceptionProductGate.ps1": (
|
||||
"experiments/perception/worker/Invoke-E40PerceptionProductGate.ps1"
|
||||
),
|
||||
}
|
||||
_GENERATED_COMPUTE_INIT = (
|
||||
'"""Minimal E40 worker projection; import contract modules explicitly."""\n'
|
||||
)
|
||||
_ACCEPTANCE_FILES = (
|
||||
"manifest.json",
|
||||
"acceptance-items.jsonl",
|
||||
"acceptance-contract.json",
|
||||
"run-report.json",
|
||||
)
|
||||
_MATERIALIZATION_FILES = ("manifest.json", "materialized-items.jsonl")
|
||||
|
||||
|
||||
class E40WorkerPackageError(RuntimeError):
|
||||
"""The E40 package source or immutable package is invalid."""
|
||||
|
||||
|
||||
def build_e40_worker_package(
|
||||
*,
|
||||
repository_root: Path,
|
||||
acceptance_root: Path,
|
||||
materialization_root: Path,
|
||||
profile_path: Path,
|
||||
output_root: Path,
|
||||
) -> Path:
|
||||
"""Build or verify one content-addressed E40 worker package."""
|
||||
|
||||
repository = repository_root.resolve(strict=True)
|
||||
profile_source = profile_path.resolve(strict=True)
|
||||
profile = _read_json(profile_source)
|
||||
if profile.get("schema_version") != E40_PROFILE_SCHEMA:
|
||||
raise E40WorkerPackageError("E40 package profile is incompatible")
|
||||
acceptance = acceptance_root.resolve(strict=True)
|
||||
materialization = materialization_root.resolve(strict=True)
|
||||
expected_ids = {
|
||||
"acceptance": profile["source"]["acceptance_result_id"],
|
||||
"materialization": profile["source"]["materialization_id"],
|
||||
}
|
||||
if (
|
||||
acceptance.name != expected_ids["acceptance"]
|
||||
or materialization.name != expected_ids["materialization"]
|
||||
):
|
||||
raise E40WorkerPackageError("E40 source identity changed")
|
||||
|
||||
sources: dict[str, Path | bytes | None] = {}
|
||||
for target, relative in _RUNTIME_FILES.items():
|
||||
source = None if relative is None else repository / relative
|
||||
if source is not None and (not source.is_file() or source.is_symlink()):
|
||||
raise E40WorkerPackageError(f"E40 runtime source is invalid: {relative}")
|
||||
sources[target] = source
|
||||
sources["profile.json"] = profile_source
|
||||
for filename in _ACCEPTANCE_FILES:
|
||||
source = acceptance / filename
|
||||
if not source.is_file() or source.is_symlink():
|
||||
raise E40WorkerPackageError("E40 acceptance artifact is invalid")
|
||||
sources[f"input/acceptance/{acceptance.name}/{filename}"] = source
|
||||
for filename in _MATERIALIZATION_FILES:
|
||||
source = materialization / filename
|
||||
if not source.is_file() or source.is_symlink():
|
||||
raise E40WorkerPackageError("E40 materialization artifact is invalid")
|
||||
sources[f"input/materialization/{materialization.name}/{filename}"] = source
|
||||
_add_materialized_evidence(
|
||||
sources,
|
||||
materialization=materialization,
|
||||
)
|
||||
_add_feature_cache(
|
||||
sources,
|
||||
acceptance=acceptance,
|
||||
materialization=materialization,
|
||||
)
|
||||
|
||||
descriptors = []
|
||||
for relative, source in sorted(sources.items()):
|
||||
payload = (
|
||||
_GENERATED_COMPUTE_INIT.encode()
|
||||
if source is None
|
||||
else source
|
||||
if isinstance(source, bytes)
|
||||
else source.read_bytes()
|
||||
)
|
||||
descriptors.append(
|
||||
{
|
||||
"path": relative,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
}
|
||||
)
|
||||
identity = {
|
||||
"schema_version": E40_PACKAGE_SCHEMA,
|
||||
"classification": ("immutable-ravnoves00-leakage-resistant-product-gate-input"),
|
||||
"source_ids": expected_ids,
|
||||
"profile_sha256": _sha256(profile_source),
|
||||
"runtime_requirements": {
|
||||
"python": "3.12",
|
||||
"numpy": "1.26+",
|
||||
},
|
||||
"build_requirements": {"pillow": "10+"},
|
||||
"artifact_paths": [row["path"] for row in descriptors],
|
||||
"source_artifacts": descriptors,
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
package_id = f"e40-worker-package-{identity_sha256}"
|
||||
output = output_root.expanduser().absolute()
|
||||
output.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
destination = output / package_id
|
||||
if destination.exists():
|
||||
validate_e40_worker_package(destination)
|
||||
return destination
|
||||
|
||||
staging = output / f".{package_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700, exist_ok=False)
|
||||
try:
|
||||
for relative, source in sources.items():
|
||||
target = staging / relative
|
||||
target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if source is None:
|
||||
target.write_text(_GENERATED_COMPUTE_INIT, encoding="utf-8")
|
||||
elif isinstance(source, bytes):
|
||||
target.write_bytes(source)
|
||||
else:
|
||||
shutil.copyfile(source, target)
|
||||
artifacts = [
|
||||
{
|
||||
"kind": relative,
|
||||
"path": relative,
|
||||
"byte_length": (staging / relative).stat().st_size,
|
||||
"sha256": _sha256(staging / relative),
|
||||
}
|
||||
for relative in sorted(sources)
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": E40_PACKAGE_SCHEMA,
|
||||
"package_id": package_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": datetime.now(UTC)
|
||||
.isoformat(timespec="milliseconds")
|
||||
.replace("+00:00", "Z"),
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
_write_json(staging / "manifest.json", manifest)
|
||||
validate_e40_worker_package(staging, allow_staging=True)
|
||||
os.replace(staging, destination)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
validate_e40_worker_package(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def validate_e40_worker_package(
|
||||
root: Path,
|
||||
*,
|
||||
allow_staging: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate package identity, exact file set, and every member digest."""
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / "manifest.json")
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
package_id = manifest.get("package_id")
|
||||
artifacts = manifest.get("artifacts")
|
||||
source_artifacts = identity.get("source_artifacts") if isinstance(identity, dict) else None
|
||||
expected_name = isinstance(package_id, str) and (
|
||||
resolved.name == package_id
|
||||
or (
|
||||
allow_staging
|
||||
and resolved.name.startswith(f".{package_id}.")
|
||||
and resolved.name.endswith(".tmp")
|
||||
)
|
||||
)
|
||||
if (
|
||||
manifest.get("schema_version") != E40_PACKAGE_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
||||
or package_id != f"e40-worker-package-{identity_sha256}"
|
||||
or not expected_name
|
||||
or not isinstance(artifacts, list)
|
||||
or not isinstance(source_artifacts, list)
|
||||
):
|
||||
raise E40WorkerPackageError("E40 worker package identity is invalid")
|
||||
expected_paths = set(identity.get("artifact_paths", []))
|
||||
bound_artifacts: dict[str, tuple[int, str]] = {}
|
||||
for row in source_artifacts:
|
||||
if (
|
||||
not isinstance(row, dict)
|
||||
or not isinstance((relative := row.get("path")), str)
|
||||
or relative in bound_artifacts
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not isinstance((byte_length := row.get("byte_length")), int)
|
||||
or byte_length < 0
|
||||
or not isinstance((sha256 := row.get("sha256")), str)
|
||||
or len(sha256) != 64
|
||||
):
|
||||
raise E40WorkerPackageError("E40 bound source artifact is invalid")
|
||||
bound_artifacts[relative] = (byte_length, sha256)
|
||||
if set(bound_artifacts) != expected_paths:
|
||||
raise E40WorkerPackageError("E40 bound artifact coverage changed")
|
||||
actual_paths = {
|
||||
path.relative_to(resolved).as_posix() for path in resolved.rglob("*") if path.is_file()
|
||||
}
|
||||
if (
|
||||
not expected_paths
|
||||
or actual_paths != expected_paths | {"manifest.json"}
|
||||
or len(artifacts) != len(expected_paths)
|
||||
):
|
||||
raise E40WorkerPackageError("E40 worker package file set changed")
|
||||
observed: set[str] = set()
|
||||
for row in artifacts:
|
||||
if not isinstance(row, dict):
|
||||
raise E40WorkerPackageError("E40 worker package artifact is invalid")
|
||||
relative = row.get("path")
|
||||
path = resolved / str(relative)
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative not in expected_paths
|
||||
or relative in observed
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
or not path.is_file()
|
||||
or path.is_symlink()
|
||||
or row.get("kind") != relative
|
||||
or bound_artifacts.get(relative)
|
||||
!= (row.get("byte_length"), row.get("sha256"))
|
||||
or row.get("byte_length") != path.stat().st_size
|
||||
or row.get("sha256") != _sha256(path)
|
||||
):
|
||||
raise E40WorkerPackageError("E40 worker package artifact changed")
|
||||
observed.add(relative)
|
||||
if observed != expected_paths:
|
||||
raise E40WorkerPackageError("E40 worker package coverage changed")
|
||||
return manifest
|
||||
|
||||
|
||||
def _add_materialized_evidence(
|
||||
sources: dict[str, Path | bytes | None],
|
||||
*,
|
||||
materialization: Path,
|
||||
) -> None:
|
||||
rows = _read_jsonl(materialization / "materialized-items.jsonl")
|
||||
if len(rows) != 486:
|
||||
raise E40WorkerPackageError("E40 materialization denominator changed")
|
||||
for row in rows:
|
||||
for descriptor_name in ("artifact", "camera_frame"):
|
||||
descriptor = row.get(descriptor_name)
|
||||
if not isinstance(descriptor, dict):
|
||||
raise E40WorkerPackageError("E40 materialization descriptor is invalid")
|
||||
relative = descriptor.get("path")
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or Path(relative).is_absolute()
|
||||
or ".." in Path(relative).parts
|
||||
):
|
||||
raise E40WorkerPackageError("E40 materialization path is invalid")
|
||||
source = materialization / relative
|
||||
if (
|
||||
not source.is_file()
|
||||
or source.is_symlink()
|
||||
or descriptor.get("byte_length") != source.stat().st_size
|
||||
or descriptor.get("sha256") != _sha256(source)
|
||||
):
|
||||
raise E40WorkerPackageError("E40 materialized evidence content changed")
|
||||
target = f"input/materialization/{materialization.name}/{relative}"
|
||||
existing = sources.get(target)
|
||||
if existing is not None and existing != source:
|
||||
raise E40WorkerPackageError("E40 package target collision")
|
||||
sources[target] = source
|
||||
|
||||
|
||||
def _add_feature_cache(
|
||||
sources: dict[str, Path | bytes | None],
|
||||
*,
|
||||
acceptance: Path,
|
||||
materialization: Path,
|
||||
) -> None:
|
||||
acceptance_rows = _read_jsonl(acceptance / "acceptance-items.jsonl")
|
||||
materialization_rows = _read_jsonl(materialization / "materialized-items.jsonl")
|
||||
acceptance_by_id = {str(row["item_id"]): row for row in acceptance_rows}
|
||||
if len(acceptance_by_id) != 486 or {str(row["item_id"]) for row in materialization_rows} != set(
|
||||
acceptance_by_id
|
||||
):
|
||||
raise E40WorkerPackageError("E40 feature-cache denominator changed")
|
||||
item_ids = [str(row["item_id"]) for row in materialization_rows]
|
||||
features = np.asarray(
|
||||
[
|
||||
_feature_vector(
|
||||
acceptance_by_id[item_id],
|
||||
row,
|
||||
materialization,
|
||||
)
|
||||
for item_id, row in zip(item_ids, materialization_rows, strict=True)
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
names = _feature_names()
|
||||
if features.shape != (486, len(names)) or not np.isfinite(features).all():
|
||||
raise E40WorkerPackageError("E40 feature-cache matrix is invalid")
|
||||
arrays_stream = io.BytesIO()
|
||||
np.savez_compressed(
|
||||
arrays_stream,
|
||||
item_ids=np.asarray(item_ids, dtype=f"<U{max(map(len, item_ids))}"),
|
||||
features=features,
|
||||
)
|
||||
arrays_payload = arrays_stream.getvalue()
|
||||
manifest = {
|
||||
"schema_version": _FEATURE_CACHE_SCHEMA,
|
||||
"materialization_id": materialization.name,
|
||||
"materialization_index_sha256": _sha256(materialization / "materialized-items.jsonl"),
|
||||
"feature_names_sha256": hashlib.sha256(_canonical_json(names)).hexdigest(),
|
||||
"items": 486,
|
||||
"dimensions": len(names),
|
||||
"arrays_path": _FEATURE_CACHE_ARRAYS,
|
||||
"arrays_byte_length": len(arrays_payload),
|
||||
"arrays_sha256": hashlib.sha256(arrays_payload).hexdigest(),
|
||||
}
|
||||
manifest_payload = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()
|
||||
prefix = f"input/materialization/{materialization.name}"
|
||||
sources[f"{prefix}/{_FEATURE_CACHE_ARRAYS}"] = arrays_payload
|
||||
sources[f"{prefix}/{_FEATURE_CACHE_MANIFEST}"] = manifest_payload
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise E40WorkerPackageError(f"JSON object expected: {path.name}")
|
||||
return value
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows = [json.loads(line) for line in path.read_text(encoding="utf-8-sig").splitlines()]
|
||||
if not all(isinstance(row, dict) for row in rows):
|
||||
raise E40WorkerPackageError(f"JSONL object expected: {path.name}")
|
||||
return rows
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
json.dump(value, stream, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--repository-root", type=Path, required=True)
|
||||
parser.add_argument("--acceptance", type=Path, required=True)
|
||||
parser.add_argument("--materialization", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = build_e40_worker_package(
|
||||
repository_root=args.repository_root,
|
||||
acceptance_root=args.acceptance,
|
||||
materialization_root=args.materialization,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(package)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the development-qualified RAVNOVES00 E40 product gate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e40_perception_product_gate import (
|
||||
build_e40_perception_product_gate,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--acceptance", type=Path, required=True)
|
||||
parser.add_argument("--materialization", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
parser.add_argument("--worker-node", default=os.environ.get("COMPUTERNAME"))
|
||||
args = parser.parse_args()
|
||||
result = build_e40_perception_product_gate(
|
||||
acceptance_root=args.acceptance,
|
||||
materialization_root=args.materialization,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
worker_node=args.worker_node,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"quality_gate_passed": result.quality_gate_passed,
|
||||
"development_cross_validation": result.report["development_cross_validation"],
|
||||
"metrics": result.report["metrics"],
|
||||
"blocking_checks": result.report["quality_gate"]["blocking_checks"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare/run the E41 truth-free predictor and visible evaluator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import platform
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from k1link.compute.e41_evaluation_boundary import (
|
||||
E41_FEATURE_MANIFEST_NAME,
|
||||
build_e41_predictor_package,
|
||||
build_e41_visible_evaluation,
|
||||
run_e41_predictor,
|
||||
)
|
||||
from k1link.compute.pipeline_telemetry import (
|
||||
JsonlPipelineTelemetrySink,
|
||||
PipelineStageOutcome,
|
||||
PipelineTelemetryEmitter,
|
||||
PipelineTelemetryIdentity,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
package = subparsers.add_parser("package")
|
||||
package.add_argument("--materialization", type=Path, required=True)
|
||||
package.add_argument("--e40-package", type=Path, required=True)
|
||||
package.add_argument("--e40-model", type=Path, required=True)
|
||||
package.add_argument("--output-root", type=Path, required=True)
|
||||
_add_telemetry_arguments(package)
|
||||
|
||||
predict = subparsers.add_parser("predict")
|
||||
predict.add_argument("--package", type=Path, required=True)
|
||||
predict.add_argument("--output-root", type=Path, required=True)
|
||||
predict.add_argument("--environment-lock", required=True)
|
||||
predict.add_argument("--python-version", required=True)
|
||||
predict.add_argument("--numpy-version", required=True)
|
||||
_add_telemetry_arguments(predict)
|
||||
|
||||
evaluate = subparsers.add_parser("evaluate")
|
||||
evaluate.add_argument("--prediction", type=Path, required=True)
|
||||
evaluate.add_argument("--acceptance", type=Path, required=True)
|
||||
evaluate.add_argument("--output-root", type=Path, required=True)
|
||||
_add_telemetry_arguments(evaluate)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.command == "package":
|
||||
emitter = _telemetry_emitter(
|
||||
args,
|
||||
source_package_id=args.e40_package.name,
|
||||
method_id="e41-truth-free-package/v1",
|
||||
)
|
||||
with _stage(emitter, "package") as outcome:
|
||||
result = build_e41_predictor_package(
|
||||
materialization_root=args.materialization,
|
||||
e40_package_root=args.e40_package,
|
||||
e40_model_path=args.e40_model,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
feature_manifest = json.loads(
|
||||
(result / E41_FEATURE_MANIFEST_NAME).read_text(encoding="utf-8")
|
||||
)
|
||||
outcome.output_count = int(feature_manifest["item_count"])
|
||||
payload = {"predictor_package": str(result)}
|
||||
elif args.command == "predict":
|
||||
emitter = _telemetry_emitter(
|
||||
args,
|
||||
source_package_id=args.package.name,
|
||||
method_id="frozen-e40-predictor/v1",
|
||||
)
|
||||
with _stage(emitter, "predict") as outcome:
|
||||
prediction = run_e41_predictor(
|
||||
package_root=args.package,
|
||||
output_root=args.output_root,
|
||||
runtime_identity={
|
||||
"environment_lock": args.environment_lock,
|
||||
"python": args.python_version,
|
||||
"numpy": args.numpy_version,
|
||||
},
|
||||
)
|
||||
outcome.output_count = int(prediction.manifest["item_count"])
|
||||
payload = {
|
||||
"prediction_result_id": prediction.result_id,
|
||||
"prediction_root": str(prediction.result_root),
|
||||
}
|
||||
else:
|
||||
emitter = _telemetry_emitter(
|
||||
args,
|
||||
source_package_id=args.prediction.name,
|
||||
method_id="visible-engineering-contract/v1",
|
||||
)
|
||||
with _stage(emitter, "evaluate") as outcome:
|
||||
evaluation = build_e41_visible_evaluation(
|
||||
prediction_root=args.prediction,
|
||||
acceptance_root=args.acceptance,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
outcome.output_count = int(
|
||||
evaluation.evaluation["evaluation"]["metrics"]["validation_items"]
|
||||
)
|
||||
payload = {
|
||||
"evaluation_result_id": evaluation.result_id,
|
||||
"evaluation_root": str(evaluation.result_root),
|
||||
"metrics": evaluation.evaluation["evaluation"]["metrics"],
|
||||
"blocking_checks": evaluation.evaluation["evaluation"]["blocking_checks"],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _add_telemetry_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"--telemetry-jsonl",
|
||||
type=Path,
|
||||
help="append native pipeline events to this local JSONL evidence file",
|
||||
)
|
||||
parser.add_argument("--telemetry-contour-id", default="local-compute")
|
||||
parser.add_argument("--telemetry-agent-id", default="mission-core-runner")
|
||||
parser.add_argument("--telemetry-node-id", default=platform.node() or "unknown-node")
|
||||
parser.add_argument("--telemetry-run-id")
|
||||
parser.add_argument("--telemetry-request-id")
|
||||
parser.add_argument("--telemetry-source-id", default="ravnoves00")
|
||||
|
||||
|
||||
def _telemetry_emitter(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
source_package_id: str,
|
||||
method_id: str,
|
||||
) -> PipelineTelemetryEmitter | None:
|
||||
if args.telemetry_jsonl is None:
|
||||
return None
|
||||
identity = PipelineTelemetryIdentity(
|
||||
contour_id=str(args.telemetry_contour_id),
|
||||
agent_id=str(args.telemetry_agent_id),
|
||||
node_id=str(args.telemetry_node_id),
|
||||
lab_id="E41",
|
||||
run_id=str(args.telemetry_run_id or f"e41-{uuid.uuid4().hex}"),
|
||||
request_id=(
|
||||
str(args.telemetry_request_id)
|
||||
if args.telemetry_request_id is not None
|
||||
else None
|
||||
),
|
||||
source_id=str(args.telemetry_source_id),
|
||||
source_package_id=source_package_id,
|
||||
method_id=method_id,
|
||||
)
|
||||
return PipelineTelemetryEmitter(
|
||||
identity=identity,
|
||||
sink=JsonlPipelineTelemetrySink(args.telemetry_jsonl),
|
||||
)
|
||||
|
||||
|
||||
def _stage(
|
||||
emitter: PipelineTelemetryEmitter | None,
|
||||
stage_id: str,
|
||||
) -> Any:
|
||||
if emitter is None:
|
||||
return contextlib.nullcontext(PipelineStageOutcome())
|
||||
return emitter.stage(stage_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the immutable RAVNOVES00 E41 methodology audit."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e41_methodology_audit import build_e41_methodology_audit
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--acceptance", type=Path, required=True)
|
||||
parser.add_argument("--materialization", type=Path, required=True)
|
||||
parser.add_argument("--e40-package", type=Path, required=True)
|
||||
parser.add_argument("--e40-result", type=Path, required=True)
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e41_methodology_audit(
|
||||
acceptance_root=args.acceptance,
|
||||
materialization_root=args.materialization,
|
||||
e40_package_root=args.e40_package,
|
||||
e40_result_root=args.e40_result,
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"blind_gate_eligible": result.blind_gate_eligible,
|
||||
"violations": result.report["analysis"]["policy"]["violations"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the bounded E42 predictor and PointSlab metamorphic suite."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e42_metamorphic_suite import build_e42_metamorphic_suite
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--predictor-package", type=Path, required=True)
|
||||
parser.add_argument("--e32-result", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e42_metamorphic_suite(
|
||||
predictor_package_root=args.predictor_package,
|
||||
e32_result_root=args.e32_result,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"accepted": result.accepted,
|
||||
"checks": result.report["acceptance"]["checks"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freeze the future same-K1/new-route capture protocol."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e43_future_capture_protocol import (
|
||||
build_e43_future_capture_protocol,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e43_future_capture_protocol(
|
||||
profile_path=args.profile,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"capture_exists": result.protocol["capture_exists"],
|
||||
"labels_exist": result.protocol["labels_exist"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure exact data amplification across explicit immutable LAB roots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e44_data_amplification_audit import (
|
||||
build_e44_data_amplification_audit,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--artifact-root",
|
||||
action="append",
|
||||
required=True,
|
||||
metavar="LABEL=PATH",
|
||||
)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
roots: dict[str, Path] = {}
|
||||
for value in args.artifact_root:
|
||||
label, separator, raw_path = value.partition("=")
|
||||
if not separator or label in roots:
|
||||
parser.error("--artifact-root must be a unique LABEL=PATH")
|
||||
roots[label] = Path(raw_path)
|
||||
result = build_e44_data_amplification_audit(
|
||||
artifact_roots=roots,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"analysis": result.report["analysis"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,161 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$PackageRoot,
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\e40-product-gate",
|
||||
[string]$ContainerImage = "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
[ValidateRange(1, 1000)]
|
||||
[int]$FreeGiBFloor = 300
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-LastExitCode([string]$Operation) {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$Operation failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Resolve-DDirectory([string]$Path, [string]$Label) {
|
||||
$item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force
|
||||
$root = [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\")
|
||||
if (
|
||||
-not $item.PSIsContainer -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or
|
||||
$root -ine "D:"
|
||||
) {
|
||||
throw "$Label must be a real D: directory"
|
||||
}
|
||||
return $item.FullName
|
||||
}
|
||||
|
||||
function Convert-ToDockerPath([string]$Path) {
|
||||
return $Path.Replace("\", "/")
|
||||
}
|
||||
|
||||
function Assert-FreeSpace([string]$Phase) {
|
||||
$free = [int64](Get-PSDrive -Name D).Free
|
||||
$floor = [int64]$FreeGiBFloor * 1GB
|
||||
Write-Host (
|
||||
"DISK_GUARD PHASE={0} DRIVE=D FREE_BYTES={1} FREE_GIB={2} FLOOR_GIB={3}" -f
|
||||
$Phase, $free, [math]::Round($free / 1GB, 3), $FreeGiBFloor
|
||||
)
|
||||
if ($free -lt ($floor + 1GB)) {
|
||||
throw "D: lacks the guarded E40 reserve during $Phase"
|
||||
}
|
||||
return $free
|
||||
}
|
||||
|
||||
$package = Resolve-DDirectory $PackageRoot "E40 package"
|
||||
$packageManifestPath = Join-Path $package "manifest.json"
|
||||
if (-not (Test-Path -LiteralPath $packageManifestPath -PathType Leaf)) {
|
||||
throw "E40 package manifest is missing"
|
||||
}
|
||||
$packageManifest = Get-Content -LiteralPath $packageManifestPath -Raw |
|
||||
ConvertFrom-Json
|
||||
if (
|
||||
$packageManifest.schema_version -ne "missioncore.e40-worker-package/v1" -or
|
||||
$packageManifest.package_id -ne (Split-Path $package -Leaf) -or
|
||||
$packageManifest.package_id -notmatch "^e40-worker-package-[a-f0-9]{64}$"
|
||||
) {
|
||||
throw "E40 package manifest is incompatible"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $OutputRoot)) {
|
||||
$null = New-Item -ItemType Directory -Path $OutputRoot
|
||||
}
|
||||
$output = Resolve-DDirectory $OutputRoot "E40 output root"
|
||||
$freeBefore = Assert-FreeSpace "preflight"
|
||||
|
||||
& docker image inspect $ContainerImage *> $null
|
||||
Assert-LastExitCode "Pinned E40 container image inspection"
|
||||
|
||||
$dockerPackage = Convert-ToDockerPath $package
|
||||
$dockerOutput = Convert-ToDockerPath $output
|
||||
$packageName = Split-Path $package -Leaf
|
||||
$containerPackage = "/opt/e40-input/$packageName"
|
||||
$packageValidator = (
|
||||
"{0}/runtime/validate_e40_worker_package.py" -f $containerPackage
|
||||
)
|
||||
$validationCommand = @(
|
||||
"run", "--rm",
|
||||
"--name", "ndc-mission-core-e40-package-validation",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "32",
|
||||
"--memory", "128m",
|
||||
"--memory-swap", "128m",
|
||||
"--cpus", "1",
|
||||
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
$packageValidator,
|
||||
$containerPackage
|
||||
)
|
||||
& docker @validationCommand
|
||||
Assert-LastExitCode "Independent E40 package integrity verification"
|
||||
|
||||
$command = @(
|
||||
"run", "--rm",
|
||||
"--name", "ndc-mission-core-e40-product-gate",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--security-opt", "no-new-privileges:true",
|
||||
"--cap-drop", "ALL",
|
||||
"--pids-limit", "128",
|
||||
"--memory", "1g",
|
||||
"--memory-swap", "1g",
|
||||
"--cpus", "2",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=64m",
|
||||
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e", ("PYTHONPATH={0}/runtime" -f $containerPackage),
|
||||
"-e", ("E40_WORKER_NODE={0}" -f $env:COMPUTERNAME),
|
||||
"-v", ("{0}:{1}:ro" -f $dockerPackage, $containerPackage),
|
||||
"-v", ("{0}:/output:rw" -f $dockerOutput),
|
||||
"--entrypoint", "python3",
|
||||
$ContainerImage,
|
||||
("{0}/runtime/run_e40_perception_product_gate.py" -f $containerPackage),
|
||||
"--package", $containerPackage,
|
||||
"--output-root", "/output"
|
||||
)
|
||||
|
||||
Write-Output ("PACKAGE_ID={0}" -f $packageManifest.package_id)
|
||||
Write-Output ("PACKAGE_IDENTITY_SHA256={0}" -f $packageManifest.identity_sha256)
|
||||
Write-Output ("CONTAINER_IMAGE={0}" -f $ContainerImage)
|
||||
& docker @command
|
||||
Assert-LastExitCode "E40 perception product gate"
|
||||
|
||||
$matches = @(
|
||||
Get-ChildItem -LiteralPath $output -Directory -Filter "e40-perception-product-gate-*" |
|
||||
Where-Object {
|
||||
$manifestPath = Join-Path $_.FullName "manifest.json"
|
||||
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
|
||||
return $false
|
||||
}
|
||||
$manifest = Get-Content -LiteralPath $manifestPath -Raw |
|
||||
ConvertFrom-Json
|
||||
return (
|
||||
$manifest.schema_version -eq
|
||||
"missioncore.e40-perception-product-gate/v1" -and
|
||||
$manifest.acceptance_state -eq
|
||||
"completed-leakage-resistant-product-gate" -and
|
||||
$manifest.identity.execution.worker_node -eq $env:COMPUTERNAME
|
||||
)
|
||||
}
|
||||
)
|
||||
if ($matches.Count -ne 1) {
|
||||
throw "E40 immutable result could not be resolved uniquely"
|
||||
}
|
||||
$resultRoot = $matches[0].FullName
|
||||
$resultManifest = Get-Content -LiteralPath (
|
||||
Join-Path $resultRoot "manifest.json"
|
||||
) -Raw | ConvertFrom-Json
|
||||
$freeAfter = Assert-FreeSpace "completed"
|
||||
Write-Output ("RESULT_ROOT={0}" -f $resultRoot)
|
||||
Write-Output ("RESULT_ID={0}" -f $resultManifest.result_id)
|
||||
Write-Output ("QUALITY_GATE_PASSED={0}" -f $resultManifest.quality_gate_passed)
|
||||
Write-Output ("DISK_FREE_BYTES_BEFORE={0}" -f $freeBefore)
|
||||
Write-Output ("DISK_FREE_BYTES_AFTER={0}" -f $freeAfter)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Execute the packaged E40 product gate in the pinned Worker 006 container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e40_perception_product_gate import (
|
||||
build_e40_perception_product_gate,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--package", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
package = args.package.resolve(strict=True)
|
||||
package_manifest = json.loads(
|
||||
(package / "manifest.json").read_text(encoding="utf-8-sig")
|
||||
)
|
||||
profile = json.loads((package / "profile.json").read_text(encoding="utf-8"))
|
||||
acceptance_id = profile["source"]["acceptance_result_id"]
|
||||
materialization_id = profile["source"]["materialization_id"]
|
||||
result = build_e40_perception_product_gate(
|
||||
acceptance_root=package / "input" / "acceptance" / acceptance_id,
|
||||
materialization_root=(package / "input" / "materialization" / materialization_id),
|
||||
profile_path=package / "profile.json",
|
||||
output_root=args.output_root,
|
||||
worker_node=os.environ.get("E40_WORKER_NODE"),
|
||||
execution_package={
|
||||
"mode": "verified-worker-package",
|
||||
"package_id": package_manifest["package_id"],
|
||||
"identity_sha256": package_manifest["identity_sha256"],
|
||||
},
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"quality_gate_passed": result.quality_gate_passed,
|
||||
"metrics": result.report["metrics"],
|
||||
"blocking_checks": result.report["quality_gate"]["blocking_checks"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independently validate an E40 package before importing package code."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
def _descriptors(
|
||||
rows: object,
|
||||
*,
|
||||
require_kind: bool,
|
||||
) -> dict[str, tuple[int, str]]:
|
||||
if not isinstance(rows, list):
|
||||
raise SystemExit("E40 package artifact catalog is missing")
|
||||
result: dict[str, tuple[int, str]] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise SystemExit("E40 package artifact descriptor is invalid")
|
||||
relative = row.get("path")
|
||||
byte_length = row.get("byte_length")
|
||||
sha256 = row.get("sha256")
|
||||
if (
|
||||
not isinstance(relative, str)
|
||||
or relative in result
|
||||
or pathlib.PurePosixPath(relative).is_absolute()
|
||||
or ".." in pathlib.PurePosixPath(relative).parts
|
||||
or not isinstance(byte_length, int)
|
||||
or byte_length < 0
|
||||
or not isinstance(sha256, str)
|
||||
or len(sha256) != 64
|
||||
or (require_kind and row.get("kind") != relative)
|
||||
):
|
||||
raise SystemExit("E40 package artifact descriptor is invalid")
|
||||
result[relative] = (byte_length, sha256)
|
||||
return result
|
||||
|
||||
|
||||
def validate(root_argument: str) -> None:
|
||||
root = pathlib.Path(root_argument).resolve(strict=True)
|
||||
manifest = json.loads(
|
||||
(root / "manifest.json").read_text(encoding="utf-8-sig")
|
||||
)
|
||||
identity = manifest.get("identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
package_id = manifest.get("package_id")
|
||||
if (
|
||||
manifest.get("schema_version") != "missioncore.e40-worker-package/v1"
|
||||
or not isinstance(identity, dict)
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(
|
||||
json.dumps(
|
||||
identity,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode()
|
||||
).hexdigest()
|
||||
!= identity_sha256
|
||||
or package_id != f"e40-worker-package-{identity_sha256}"
|
||||
or root.name != package_id
|
||||
):
|
||||
raise SystemExit("E40 package identity verification failed")
|
||||
|
||||
bound = _descriptors(
|
||||
identity.get("source_artifacts"),
|
||||
require_kind=False,
|
||||
)
|
||||
catalog = _descriptors(manifest.get("artifacts"), require_kind=True)
|
||||
if bound != catalog or set(identity.get("artifact_paths", [])) != set(bound):
|
||||
raise SystemExit("E40 package artifact binding verification failed")
|
||||
actual = {
|
||||
path.relative_to(root).as_posix()
|
||||
for path in root.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
if actual != set(bound) | {"manifest.json"}:
|
||||
raise SystemExit("E40 package file set verification failed")
|
||||
for relative, (byte_length, sha256) in bound.items():
|
||||
path = root / relative
|
||||
payload = path.read_bytes()
|
||||
if (
|
||||
path.is_symlink()
|
||||
or len(payload) != byte_length
|
||||
or hashlib.sha256(payload).hexdigest() != sha256
|
||||
):
|
||||
raise SystemExit(
|
||||
f"E40 package member verification failed: {relative}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit("usage: validate_e40_worker_package.py PACKAGE_ROOT")
|
||||
validate(sys.argv[1])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user