feat(perception): stabilize pre-capture methodology
This commit is contained in:
DCCONSTRUCTIONS
@@ -26,7 +26,9 @@ def _endpoint(router: APIRouter, path: str) -> object:
|
||||
def test_advanced_catalogs_are_empty_when_not_configured() -> None:
|
||||
router = build_advanced_laboratory_router()
|
||||
|
||||
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"):
|
||||
for name in (
|
||||
"e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40"
|
||||
):
|
||||
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
||||
catalog = route(limit=1) # type: ignore[operator]
|
||||
assert catalog == {
|
||||
@@ -50,7 +52,8 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
||||
e37 = tmp_path / "e37"
|
||||
e38 = tmp_path / "e38"
|
||||
e39 = tmp_path / "e39"
|
||||
for root in (e31, e32, e33, e34, e35, e37, e38, e39):
|
||||
e40 = tmp_path / "e40"
|
||||
for root in (e31, e32, e33, e34, e35, e37, e38, e39, e40):
|
||||
root.mkdir()
|
||||
(e31 / f"e31-source-qualification-{'1' * 64}").mkdir()
|
||||
(e32 / f"e32-track-geometry-{'2' * 64}").mkdir()
|
||||
@@ -60,6 +63,7 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
||||
(e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir()
|
||||
(e38 / f"e38-perception-baseline-{'8' * 64}").mkdir()
|
||||
(e39 / f"e39-perception-refinement-{'9' * 64}").mkdir()
|
||||
(e40 / f"e40-perception-product-gate-{'a' * 64}").mkdir()
|
||||
router = build_advanced_laboratory_router(
|
||||
e31_root_provider=lambda: e31,
|
||||
e32_root_provider=lambda: e32,
|
||||
@@ -69,9 +73,12 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
|
||||
e37_root_provider=lambda: e37,
|
||||
e38_root_provider=lambda: e38,
|
||||
e39_root_provider=lambda: e39,
|
||||
e40_root_provider=lambda: e40,
|
||||
)
|
||||
|
||||
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39"):
|
||||
for name in (
|
||||
"e31", "e32", "e33", "e34", "e35", "e37", "e38", "e39", "e40"
|
||||
):
|
||||
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
|
||||
catalog = route(limit=1) # type: ignore[operator]
|
||||
assert catalog["configured"] is True
|
||||
@@ -370,6 +377,148 @@ def test_e39_catalog_projects_development_cv_and_sealed_validation(
|
||||
assert item["access"] == "read-only"
|
||||
|
||||
|
||||
def test_e40_catalog_projects_dual_cv_and_sealed_product_gate(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
) -> None:
|
||||
result_id = f"e40-perception-product-gate-{'a' * 64}"
|
||||
root = tmp_path / "e40"
|
||||
candidate = root / result_id
|
||||
candidate.mkdir(parents=True)
|
||||
(candidate / "manifest.json").write_text("{}", encoding="utf-8")
|
||||
authority = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
dimension = {
|
||||
"correct": 132,
|
||||
"incorrect": 14,
|
||||
"total": 146,
|
||||
"accuracy": 0.90411,
|
||||
"target": 0.9,
|
||||
"passed": True,
|
||||
"confusion": [],
|
||||
"by_stratum": {},
|
||||
}
|
||||
development_dimension = {
|
||||
"correct": 309,
|
||||
"incorrect": 31,
|
||||
"total": 340,
|
||||
"accuracy": 0.908824,
|
||||
"target": 0.9,
|
||||
"passed": True,
|
||||
}
|
||||
protocol = {
|
||||
"items": 340,
|
||||
"fold_sizes": {"0": 68, "1": 68, "2": 68, "3": 68, "4": 68},
|
||||
"dimensions": {
|
||||
"presence": development_dimension,
|
||||
"geometry_association": development_dimension,
|
||||
"freshness": {
|
||||
**development_dimension,
|
||||
"correct": 325,
|
||||
"incorrect": 15,
|
||||
"accuracy": 0.955882,
|
||||
},
|
||||
},
|
||||
"passed": True,
|
||||
}
|
||||
result = SimpleNamespace(
|
||||
result_id=result_id,
|
||||
manifest={
|
||||
"created_at_utc": "2026-07-28T08:30:00Z",
|
||||
"identity": {
|
||||
"source": {
|
||||
"session_id": "20260720T065719Z_viewer_live",
|
||||
"display_name": "RAVNOVES00",
|
||||
},
|
||||
"profile": {
|
||||
"profile_id": (
|
||||
"e40-ravnoves00-leakage-resistant-product-gate/v1"
|
||||
),
|
||||
},
|
||||
"execution": {
|
||||
"worker_node": "DESKTOP-OPJ8J04",
|
||||
},
|
||||
},
|
||||
},
|
||||
report={
|
||||
"status": "measured-leakage-resistant-product-gate",
|
||||
"development_cross_validation": {
|
||||
"strategy": "dual-leakage-resistant-development-five-fold",
|
||||
"seed": "e40-development-cv-v1",
|
||||
"folds": 5,
|
||||
"items": 340,
|
||||
"validation_labels_used": False,
|
||||
"protocols": {
|
||||
"contiguous-source-time-five-fold": protocol,
|
||||
"whole-track-or-scene-window-five-fold": protocol,
|
||||
},
|
||||
"passed": True,
|
||||
},
|
||||
"metrics": {
|
||||
"development_items": 340,
|
||||
"validation_items": 146,
|
||||
"terminal_outcomes": 146,
|
||||
"accounting_fraction": 1.0,
|
||||
"false_free_claims": 0,
|
||||
"high_severity_failures": 0,
|
||||
"dimensions": {
|
||||
"presence": dimension,
|
||||
"geometry_association": dimension,
|
||||
"freshness": {
|
||||
**dimension,
|
||||
"correct": 140,
|
||||
"incorrect": 6,
|
||||
"accuracy": 0.958904,
|
||||
},
|
||||
},
|
||||
},
|
||||
"quality_gate": {
|
||||
"passed": True,
|
||||
"blocking_checks": [],
|
||||
},
|
||||
"method": {
|
||||
"summary": "conservative policy plus camera-only softmax",
|
||||
"selection": "dual grouped development cross-validation",
|
||||
"dimension_projection": "presence plus immutable stratum",
|
||||
},
|
||||
"decision": {
|
||||
"product_gate_measured": True,
|
||||
"accepted_for_ravnoves00_product_track": True,
|
||||
},
|
||||
"limitations": ["RAVNOVES00 source scoped"],
|
||||
"authority": authority,
|
||||
},
|
||||
)
|
||||
|
||||
def fake_read(
|
||||
root_text: str,
|
||||
signature: tuple[int, ...],
|
||||
) -> SimpleNamespace:
|
||||
assert root_text == str(candidate.resolve())
|
||||
assert signature
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(advanced_api, "_read_e40_cached", fake_read)
|
||||
router = build_advanced_laboratory_router(
|
||||
e40_root_provider=lambda: root,
|
||||
)
|
||||
route = _endpoint(router, "/api/v1/laboratory/e40/results")
|
||||
catalog = route(limit=1) # type: ignore[operator]
|
||||
|
||||
assert catalog["candidate_total"] == 1
|
||||
assert catalog["invalid_total"] == 0
|
||||
item = catalog["items"][0]
|
||||
assert item["development_cross_validation"]["passed"] is True
|
||||
assert len(item["development_cross_validation"]["protocols"]) == 2
|
||||
assert item["metrics"]["dimensions"]["presence"]["accuracy"] == 0.90411
|
||||
assert item["metrics"]["high_severity_failures"] == 0
|
||||
assert item["quality_gate_passed"] is True
|
||||
assert item["authority"] == authority
|
||||
assert item["access"] == "read-only"
|
||||
|
||||
|
||||
def test_e35_catalog_projects_recovery_and_review(
|
||||
tmp_path: Path,
|
||||
monkeypatch: MonkeyPatch,
|
||||
|
||||
@@ -40,6 +40,7 @@ def test_contour_store_migrates_worker_006_as_first_configuration(
|
||||
assert contours[0].contour_id == "worker-006"
|
||||
assert contours[0].telemetry_mode == "agent-mqtt"
|
||||
assert contours[0].telemetry_poll_interval_seconds == 3
|
||||
assert contours[0].mqtt_publish_interval_seconds == 2
|
||||
assert not store.path.exists()
|
||||
|
||||
|
||||
@@ -67,11 +68,13 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No
|
||||
mqtt_host="192.0.2.5",
|
||||
mqtt_port=1883,
|
||||
telemetry_poll_interval_seconds=1,
|
||||
mqtt_publish_interval_seconds=4,
|
||||
),
|
||||
)
|
||||
|
||||
assert updated.display_name == "Field Worker 01"
|
||||
assert updated.telemetry_poll_interval_seconds == 1
|
||||
assert updated.mqtt_publish_interval_seconds == 4
|
||||
assert updated.revision == 1
|
||||
assert stat.S_IMODE(store.path.stat().st_mode) == 0o600
|
||||
assert len(store.list_contours()) == 2
|
||||
@@ -89,6 +92,7 @@ def test_contour_store_creates_and_updates_private_catalog(tmp_path: Path) -> No
|
||||
mqtt_host="127.0.0.1",
|
||||
mqtt_port=1883,
|
||||
telemetry_poll_interval_seconds=3,
|
||||
mqtt_publish_interval_seconds=2,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -113,6 +117,7 @@ def test_contour_router_exposes_catalog_and_safe_install_contract(
|
||||
assert document["agent"]["distribution"] == "Telegraf"
|
||||
assert "MQTT password" in document["command"]
|
||||
assert "password" not in document["agent"]["environment"]
|
||||
assert document["agent"]["environment"]["MISSIONCORE_TELEMETRY_INTERVAL"] == "2s"
|
||||
assert document["ready"] is False
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.compute.e40_perception_product_gate import (
|
||||
_FIXED_PRESENCE,
|
||||
_LABELS,
|
||||
_feature_names,
|
||||
_predict_product_presence,
|
||||
_project_dimensions,
|
||||
_result_content_identity,
|
||||
_train_softmax,
|
||||
)
|
||||
|
||||
|
||||
def test_e40_feature_contract_excludes_route_identity() -> None:
|
||||
names = _feature_names()
|
||||
assert len(names) == 125
|
||||
assert len(names) == len(set(names))
|
||||
assert not any(
|
||||
token in name
|
||||
for name in names
|
||||
for token in (
|
||||
"source_frame",
|
||||
"session_seconds",
|
||||
"review_ordinal",
|
||||
"track_id",
|
||||
"map_xyz",
|
||||
)
|
||||
)
|
||||
assert "bbox_area" in names
|
||||
assert "image_edge_p90" in names
|
||||
assert "support_occupied_fraction" in names
|
||||
|
||||
|
||||
def test_e40_fixed_strata_are_conservative_product_states() -> None:
|
||||
assert _FIXED_PRESENCE == {
|
||||
"agree": "object-present",
|
||||
"conflict": "background-or-noise",
|
||||
"geometry-only": "occupied-environment",
|
||||
"unknown": "object-present",
|
||||
}
|
||||
median = np.zeros(2)
|
||||
scale = np.ones(2)
|
||||
weights = np.zeros((3, len(_LABELS)))
|
||||
for stratum, expected in _FIXED_PRESENCE.items():
|
||||
assert _predict_product_presence(
|
||||
stratum=stratum,
|
||||
features=np.ones(2),
|
||||
median=median,
|
||||
scale=scale,
|
||||
weights=weights,
|
||||
clip=10.0,
|
||||
) == (expected, 1.0)
|
||||
assert _project_dimensions("geometry-only", "occupied-environment") == {
|
||||
"presence": "occupied-environment",
|
||||
"geometry_association": "independent-occupied",
|
||||
"freshness": "current",
|
||||
}
|
||||
|
||||
|
||||
def test_e40_camera_only_softmax_is_deterministic() -> None:
|
||||
matrix = np.asarray(
|
||||
[
|
||||
[-2.0, -1.0],
|
||||
[-1.0, -2.0],
|
||||
[1.0, 2.0],
|
||||
[2.0, 1.0],
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
labels = np.asarray(
|
||||
[
|
||||
_LABELS.index("background-or-noise"),
|
||||
_LABELS.index("background-or-noise"),
|
||||
_LABELS.index("object-present"),
|
||||
_LABELS.index("object-present"),
|
||||
],
|
||||
dtype=np.int64,
|
||||
)
|
||||
first = _train_softmax(
|
||||
matrix,
|
||||
labels,
|
||||
l2=0.01,
|
||||
steps=120,
|
||||
learning_rate=0.03,
|
||||
)
|
||||
second = _train_softmax(
|
||||
matrix,
|
||||
labels,
|
||||
l2=0.01,
|
||||
steps=120,
|
||||
learning_rate=0.03,
|
||||
)
|
||||
assert np.array_equal(first, second)
|
||||
assert (
|
||||
_predict_product_presence(
|
||||
stratum="camera-only",
|
||||
features=np.asarray([1.5, 1.5]),
|
||||
median=np.zeros(2),
|
||||
scale=np.ones(2),
|
||||
weights=first,
|
||||
clip=10.0,
|
||||
)[0]
|
||||
== "object-present"
|
||||
)
|
||||
|
||||
|
||||
def test_e40_result_content_identity_changes_with_every_output() -> None:
|
||||
predictions = [{"sequence": 1, "prediction": {"presence": "object-present"}}]
|
||||
model = {"weights": [1.0]}
|
||||
report = {"quality_gate": {"passed": False}}
|
||||
baseline = _result_content_identity(
|
||||
predictions=predictions,
|
||||
model=model,
|
||||
report=report,
|
||||
)
|
||||
assert baseline != _result_content_identity(
|
||||
predictions=[{"sequence": 1, "prediction": {"presence": "background-or-noise"}}],
|
||||
model=model,
|
||||
report=report,
|
||||
)
|
||||
assert baseline != _result_content_identity(
|
||||
predictions=predictions,
|
||||
model={"weights": [2.0]},
|
||||
report=report,
|
||||
)
|
||||
assert baseline != _result_content_identity(
|
||||
predictions=predictions,
|
||||
model=model,
|
||||
report={"quality_gate": {"passed": True}},
|
||||
)
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _module() -> object:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "prepare_e40_worker_package.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("e40_worker_package_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_e40_package_contains_bound_product_gate_input(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
acceptance = (
|
||||
repository
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e37"
|
||||
/ "results"
|
||||
/ (
|
||||
"e37-ravnoves-acceptance-"
|
||||
"01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344"
|
||||
)
|
||||
)
|
||||
materialization = (
|
||||
repository
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e30"
|
||||
/ "materializations"
|
||||
/ ("e30-materialization-841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a")
|
||||
)
|
||||
package = module.build_e40_worker_package(
|
||||
repository_root=repository,
|
||||
acceptance_root=acceptance,
|
||||
materialization_root=materialization,
|
||||
profile_path=(
|
||||
repository / "experiments" / "perception" / "e40_ravnoves00_product_gate_profile.json"
|
||||
),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
manifest = module.validate_e40_worker_package(package)
|
||||
assert package.name == f"e40-worker-package-{manifest['identity_sha256']}"
|
||||
assert manifest["identity"]["classification"] == (
|
||||
"immutable-ravnoves00-leakage-resistant-product-gate-input"
|
||||
)
|
||||
feature_manifest = (
|
||||
package / "input" / "materialization" / materialization.name / "e40-feature-cache.json"
|
||||
)
|
||||
assert feature_manifest.is_file()
|
||||
assert (package / "runtime" / "k1link" / "compute" / "e40_perception_product_gate.py").is_file()
|
||||
independent_validator = (
|
||||
package / "runtime" / "validate_e40_worker_package.py"
|
||||
)
|
||||
assert independent_validator.is_file()
|
||||
subprocess.run(
|
||||
[sys.executable, str(independent_validator), str(package)],
|
||||
check=True,
|
||||
)
|
||||
|
||||
profile_path = package / "profile.json"
|
||||
profile_path.write_text("{}\n", encoding="utf-8")
|
||||
package_manifest_path = package / "manifest.json"
|
||||
package_manifest = json.loads(package_manifest_path.read_text(encoding="utf-8"))
|
||||
profile_payload = profile_path.read_bytes()
|
||||
for row in package_manifest["artifacts"]:
|
||||
if row["path"] == "profile.json":
|
||||
row["byte_length"] = len(profile_payload)
|
||||
row["sha256"] = hashlib.sha256(profile_payload).hexdigest()
|
||||
package_manifest_path.write_text(
|
||||
json.dumps(package_manifest, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with pytest.raises(module.E40WorkerPackageError, match="binding|changed"):
|
||||
module.validate_e40_worker_package(package)
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e41_evaluation_boundary import (
|
||||
E41EvaluationBoundaryError,
|
||||
evaluate_visible_engineering_contract,
|
||||
predict_from_frozen_e40_model,
|
||||
)
|
||||
|
||||
|
||||
def _model() -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.e40-development-product-model/v1",
|
||||
"feature_names": ["signal"],
|
||||
"validation_labels_used_for_training": False,
|
||||
"robust_clip": 10.0,
|
||||
"fixed_presence_by_stratum": {
|
||||
"agree": "object-present",
|
||||
"conflict": "background-or-noise",
|
||||
"geometry-only": "occupied-environment",
|
||||
"unknown": "object-present",
|
||||
},
|
||||
"dimension_projection": "source-stratum-plus-presence/v1",
|
||||
"classifier": {
|
||||
"type": "deterministic-softmax",
|
||||
"labels": [
|
||||
"background-or-noise",
|
||||
"object-present",
|
||||
"occupied-environment",
|
||||
],
|
||||
"weights": [
|
||||
[-2.0, 2.0, 0.0],
|
||||
[0.0, 0.0, 0.0],
|
||||
],
|
||||
"scaler": {
|
||||
"median": [0.0],
|
||||
"scale": [1.0],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_e41_predictor_output_is_truth_free_and_deterministic() -> None:
|
||||
items = [
|
||||
{
|
||||
"schema_version": "missioncore.e41-predictor-item/v1",
|
||||
"sequence": 0,
|
||||
"item_id": "a",
|
||||
"source_stratum": "camera-only",
|
||||
},
|
||||
{
|
||||
"schema_version": "missioncore.e41-predictor-item/v1",
|
||||
"sequence": 1,
|
||||
"item_id": "b",
|
||||
"source_stratum": "geometry-only",
|
||||
},
|
||||
]
|
||||
matrix = np.asarray([[1.0], [0.0]], dtype=np.float64)
|
||||
first = predict_from_frozen_e40_model(
|
||||
items=items,
|
||||
feature_names=["signal"],
|
||||
feature_matrix=matrix,
|
||||
model=_model(),
|
||||
)
|
||||
second = predict_from_frozen_e40_model(
|
||||
items=items,
|
||||
feature_names=["signal"],
|
||||
feature_matrix=matrix,
|
||||
model=_model(),
|
||||
)
|
||||
|
||||
assert first == second
|
||||
assert first[0]["prediction"]["presence"] == "object-present"
|
||||
assert first[1]["prediction"]["presence"] == "occupied-environment"
|
||||
assert not any(
|
||||
forbidden in row
|
||||
for row in first
|
||||
for forbidden in ("reference", "scored", "severity", "split", "truth")
|
||||
)
|
||||
|
||||
|
||||
def test_e41_predictor_rejects_truth_bearing_item_metadata() -> None:
|
||||
with pytest.raises(E41EvaluationBoundaryError, match="truth/evaluation"):
|
||||
predict_from_frozen_e40_model(
|
||||
items=[
|
||||
{
|
||||
"sequence": 0,
|
||||
"item_id": "a",
|
||||
"source_stratum": "camera-only",
|
||||
"reference": {"presence": "object-present"},
|
||||
}
|
||||
],
|
||||
feature_names=["signal"],
|
||||
feature_matrix=np.asarray([[1.0]], dtype=np.float64),
|
||||
model=_model(),
|
||||
)
|
||||
|
||||
|
||||
def test_e41_visible_evaluator_joins_truth_after_prediction() -> None:
|
||||
predictions = [
|
||||
{
|
||||
"item_id": "dev",
|
||||
"prediction": {
|
||||
"presence": "object-present",
|
||||
"geometry_association": "insufficient-support",
|
||||
"freshness": "unavailable",
|
||||
},
|
||||
},
|
||||
{
|
||||
"item_id": "val",
|
||||
"prediction": {
|
||||
"presence": "object-present",
|
||||
"geometry_association": "object-associated",
|
||||
"freshness": "current",
|
||||
},
|
||||
},
|
||||
]
|
||||
acceptance = [
|
||||
{
|
||||
"item_id": "dev",
|
||||
"split": "development",
|
||||
"source_stratum": "camera-only",
|
||||
"severity": "standard",
|
||||
"reference": {
|
||||
"presence": "background-or-noise",
|
||||
"geometry_association": "rejected-nonobject",
|
||||
"freshness": "unavailable",
|
||||
},
|
||||
},
|
||||
{
|
||||
"item_id": "val",
|
||||
"split": "validation",
|
||||
"source_stratum": "agree",
|
||||
"severity": "standard",
|
||||
"reference": {
|
||||
"presence": "object-present",
|
||||
"geometry_association": "object-associated",
|
||||
"freshness": "current",
|
||||
},
|
||||
},
|
||||
]
|
||||
evaluation = evaluate_visible_engineering_contract(
|
||||
predictions=predictions,
|
||||
acceptance_rows=acceptance,
|
||||
targets={
|
||||
"presence_target": 0.9,
|
||||
"geometry_association_target": 0.9,
|
||||
"freshness_target": 0.9,
|
||||
},
|
||||
label_provenance={
|
||||
"engineering_items": 2,
|
||||
"human_exception_items": 0,
|
||||
"independent_ground_truth": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert evaluation["metrics"]["validation_items"] == 1
|
||||
assert evaluation["metrics"]["dimensions"]["presence"]["accuracy"] == 1.0
|
||||
assert evaluation["engineering_contract_targets_reached"] is True
|
||||
assert evaluation["blind_gate_eligible"] is False
|
||||
assert evaluation["label_provenance"]["independent_accuracy_authority"] is False
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.compute.e41_methodology_audit import analyze_e41_methodology
|
||||
|
||||
|
||||
def _acceptance(
|
||||
item_id: str,
|
||||
*,
|
||||
split: str,
|
||||
frame: int,
|
||||
stratum: str = "camera-only",
|
||||
presence: str = "object-present",
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": item_id,
|
||||
"split": split,
|
||||
"source_frame_index": frame,
|
||||
"source_stratum": stratum,
|
||||
"reference": {
|
||||
"presence": presence,
|
||||
"geometry_association": "insufficient-support",
|
||||
"freshness": "unavailable",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _materialization(item_id: str, *, track_id: int | None) -> dict[str, object]:
|
||||
return {
|
||||
"item_id": item_id,
|
||||
"e29_snapshot": {
|
||||
"track_id": track_id,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_e41_detects_split_leakage_and_prediction_truth_colocation() -> None:
|
||||
acceptance = [
|
||||
_acceptance("dev-a", split="development", frame=100, presence="object-present"),
|
||||
_acceptance(
|
||||
"dev-b",
|
||||
split="development",
|
||||
frame=101,
|
||||
presence="background-or-noise",
|
||||
),
|
||||
_acceptance("val-a", split="validation", frame=100, presence="object-present"),
|
||||
_acceptance("val-b", split="validation", frame=149, presence="object-present"),
|
||||
]
|
||||
materialization = [
|
||||
_materialization("dev-a", track_id=7),
|
||||
_materialization("dev-b", track_id=8),
|
||||
_materialization("val-a", track_id=7),
|
||||
_materialization("val-b", track_id=None),
|
||||
]
|
||||
names = ["image_luma_mean", "stratum=camera-only", "source_frame_index"]
|
||||
matrix = np.asarray(
|
||||
[
|
||||
[0.1, 1.0, 100.0],
|
||||
[0.9, 1.0, 101.0],
|
||||
[0.2, 1.0, 100.0],
|
||||
[0.3, 1.0, 149.0],
|
||||
],
|
||||
dtype=np.float64,
|
||||
)
|
||||
report = analyze_e41_methodology(
|
||||
acceptance_rows=acceptance,
|
||||
materialization_rows=materialization,
|
||||
feature_item_ids=["dev-a", "dev-b", "val-a", "val-b"],
|
||||
feature_names=names,
|
||||
feature_matrix=matrix,
|
||||
e40_model={
|
||||
"feature_names": names,
|
||||
"camera_only_training_items": 2,
|
||||
"dimension_projection": "source-stratum-plus-presence/v1",
|
||||
},
|
||||
e40_report={
|
||||
"status": "measured-leakage-resistant-product-gate",
|
||||
"execution": {"class": "sealed-validation-evaluation"},
|
||||
"metrics": {
|
||||
"dimensions": {
|
||||
"presence": {"accuracy": 0.5},
|
||||
"geometry_association": {"accuracy": 0.5},
|
||||
}
|
||||
},
|
||||
},
|
||||
e40_predictions=[
|
||||
{
|
||||
"item_id": "val-a",
|
||||
"prediction": {"presence": "object-present"},
|
||||
"reference": {"presence": "object-present"},
|
||||
"scored": True,
|
||||
}
|
||||
],
|
||||
label_provenance={
|
||||
"engineering_items": 4,
|
||||
"human_exception_items": 0,
|
||||
"independent_ground_truth": False,
|
||||
},
|
||||
time_block_frames=50,
|
||||
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
||||
)
|
||||
|
||||
assert report["split_leakage"]["exact_source_frames"]["count"] == 1
|
||||
assert report["split_leakage"]["track_ids"]["count"] == 1
|
||||
assert report["split_leakage"]["time_blocks"]["count"] == 1
|
||||
assert report["split_leakage"]["whole_track_or_scene_groups"]["count"] == 1
|
||||
assert report["features"]["camera_only_training_items"] == 2
|
||||
assert report["features"]["forbidden_features"] == ["source_frame_index"]
|
||||
assert report["predictor_evaluator_boundary"]["physically_separated"] is False
|
||||
assert report["metric_semantics"]["dimensions_independently_inferred"] is False
|
||||
assert report["policy"]["blind_gate_eligible"] is False
|
||||
assert report["policy"]["violations"] == [
|
||||
"labels-are-not-independent-ground-truth",
|
||||
"development-validation-source-groups-overlap",
|
||||
"prediction-and-evaluation-concerns-are-co-located",
|
||||
"historical-e40-still-contains-blind-or-product-gate-claims",
|
||||
"forbidden-identity-feature-detected",
|
||||
]
|
||||
|
||||
|
||||
def test_e41_accepts_a_clean_separated_methodology_contract() -> None:
|
||||
acceptance = [
|
||||
_acceptance("dev-a", split="development", frame=10),
|
||||
_acceptance(
|
||||
"dev-b",
|
||||
split="development",
|
||||
frame=11,
|
||||
presence="background-or-noise",
|
||||
),
|
||||
_acceptance("val-a", split="validation", frame=210),
|
||||
]
|
||||
materialization = [
|
||||
_materialization("dev-a", track_id=1),
|
||||
_materialization("dev-b", track_id=2),
|
||||
_materialization("val-a", track_id=9),
|
||||
]
|
||||
names = ["image_luma_mean", "support_occupied_fraction"]
|
||||
report = analyze_e41_methodology(
|
||||
acceptance_rows=acceptance,
|
||||
materialization_rows=materialization,
|
||||
feature_item_ids=["dev-a", "dev-b", "val-a"],
|
||||
feature_names=names,
|
||||
feature_matrix=np.asarray(
|
||||
[
|
||||
[0.1, 0.2],
|
||||
[0.9, 0.8],
|
||||
[0.4, 0.3],
|
||||
],
|
||||
dtype=np.float64,
|
||||
),
|
||||
e40_model={
|
||||
"feature_names": names,
|
||||
"camera_only_training_items": 2,
|
||||
"dimension_projection": "independent-task-heads/v1",
|
||||
},
|
||||
e40_report={
|
||||
"status": "source-scoped-visible-evaluation",
|
||||
"metrics": {
|
||||
"dimensions": {
|
||||
"presence": {"accuracy": 0.8},
|
||||
"geometry_association": {"accuracy": 0.7},
|
||||
}
|
||||
},
|
||||
},
|
||||
e40_predictions=[
|
||||
{
|
||||
"item_id": "val-a",
|
||||
"prediction": {"presence": "object-present"},
|
||||
}
|
||||
],
|
||||
label_provenance={
|
||||
"engineering_items": 0,
|
||||
"human_exception_items": 3,
|
||||
"independent_ground_truth": True,
|
||||
},
|
||||
time_block_frames=50,
|
||||
forbidden_feature_tokens=("source_frame", "track_id", "path"),
|
||||
)
|
||||
|
||||
assert report["features"]["forbidden_features"] == []
|
||||
assert report["predictor_evaluator_boundary"]["physically_separated"] is True
|
||||
assert report["policy"]["violations"] == []
|
||||
assert report["policy"]["blind_gate_eligible"] is True
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
|
||||
from k1link.compute.e42_metamorphic_suite import (
|
||||
_point_slab_signature,
|
||||
_predictor_metamorphics,
|
||||
)
|
||||
from k1link.compute.track_geometry import PointSlab
|
||||
|
||||
|
||||
def _model() -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.e40-development-product-model/v1",
|
||||
"feature_names": ["signal"],
|
||||
"validation_labels_used_for_training": False,
|
||||
"robust_clip": 10.0,
|
||||
"classifier": {
|
||||
"type": "deterministic-softmax",
|
||||
"labels": [
|
||||
"background-or-noise",
|
||||
"object-present",
|
||||
"occupied-environment",
|
||||
],
|
||||
"weights": [
|
||||
[-2.0, 2.0, 0.0],
|
||||
[0.0, 0.0, 0.0],
|
||||
],
|
||||
"scaler": {
|
||||
"median": [0.0],
|
||||
"scale": [1.0],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_e42_predictor_is_invariant_to_ids_order_and_chunks() -> None:
|
||||
items = [
|
||||
{
|
||||
"schema_version": "missioncore.e41-predictor-item/v1",
|
||||
"sequence": index,
|
||||
"item_id": f"item-{index}",
|
||||
"source_stratum": "camera-only",
|
||||
}
|
||||
for index in range(12)
|
||||
]
|
||||
checks = _predictor_metamorphics(
|
||||
items=items,
|
||||
feature_names=["signal"],
|
||||
feature_matrix=np.arange(12, dtype=np.float64).reshape((-1, 1)),
|
||||
model=_model(),
|
||||
)
|
||||
|
||||
assert all(checks.values())
|
||||
|
||||
|
||||
def test_e42_point_slab_signature_is_row_order_invariant() -> None:
|
||||
slab = PointSlab(
|
||||
frame_index=1,
|
||||
source_frame_index=10,
|
||||
source_point_count=8,
|
||||
coordinate_frame="map",
|
||||
owner_keys=("track:1", "geometry:2"),
|
||||
source_indices=np.asarray([1, 7, 3], dtype="<i8"),
|
||||
points_xyz_m=np.asarray(
|
||||
[
|
||||
[1.0, 0.0, 0.0],
|
||||
[7.0, 0.0, 0.0],
|
||||
[3.0, 0.0, 0.0],
|
||||
],
|
||||
dtype="<f4",
|
||||
),
|
||||
owner_indices=np.asarray([0, 1, 0], dtype="<u4"),
|
||||
)
|
||||
order = np.asarray([2, 0, 1], dtype=np.int64)
|
||||
permuted = PointSlab(
|
||||
frame_index=slab.frame_index,
|
||||
source_frame_index=slab.source_frame_index,
|
||||
source_point_count=slab.source_point_count,
|
||||
coordinate_frame=slab.coordinate_frame,
|
||||
owner_keys=slab.owner_keys,
|
||||
source_indices=slab.source_indices[order],
|
||||
points_xyz_m=slab.points_xyz_m[order],
|
||||
owner_indices=slab.owner_indices[order],
|
||||
)
|
||||
|
||||
assert _point_slab_signature(slab) == _point_slab_signature(permuted)
|
||||
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e43_future_capture_protocol import (
|
||||
E43_CANDIDATE_SCHEMA,
|
||||
E43_CAPTURE_MANIFEST_SCHEMA,
|
||||
E43FutureCaptureProtocolError,
|
||||
assign_grouped_future_partitions,
|
||||
validate_future_capture_manifest,
|
||||
)
|
||||
|
||||
|
||||
def _protocol() -> dict[str, object]:
|
||||
return {
|
||||
"capture_contract": {
|
||||
"device_model": "XGRIDS/LixelKity-K1",
|
||||
"minimum_duration_seconds": 480,
|
||||
"maximum_duration_seconds": 900,
|
||||
"required_streams": [
|
||||
"sensor.camera.right",
|
||||
"sensor.lidar.registered-map-increment",
|
||||
"sensor.pose",
|
||||
"telemetry.pipeline",
|
||||
],
|
||||
"required_segments": [
|
||||
{
|
||||
"kind": "control-bridge",
|
||||
"minimum_duration_seconds": 60,
|
||||
},
|
||||
{
|
||||
"kind": "new-route",
|
||||
"minimum_duration_seconds": 360,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _capture_manifest() -> dict[str, object]:
|
||||
stream = {
|
||||
"available": True,
|
||||
"item_count": 10,
|
||||
"byte_length": 100,
|
||||
"sha256": "a" * 64,
|
||||
}
|
||||
return {
|
||||
"schema_version": E43_CAPTURE_MANIFEST_SCHEMA,
|
||||
"source_session_id": "future-session-001",
|
||||
"source_display_name": "RAVNOVES01",
|
||||
"operator_authorized": True,
|
||||
"device": {
|
||||
"model": "XGRIDS/LixelKity-K1",
|
||||
"device_identity_sha256": "b" * 64,
|
||||
"calibration_sha256": "c" * 64,
|
||||
"mount_identity_sha256": "d" * 64,
|
||||
"configuration_sha256": "e" * 64,
|
||||
"firmware": "3.0.2",
|
||||
},
|
||||
"capture": {
|
||||
"started_at_utc": "2026-07-28T12:00:00Z",
|
||||
"monotonic_start_seconds": 100.0,
|
||||
"monotonic_end_seconds": 700.0,
|
||||
"duration_seconds": 600.0,
|
||||
"weather": "overcast",
|
||||
"illumination": "daylight",
|
||||
"location_class": "industrial-buildings-opposite-side",
|
||||
"operator_notes": "bounded owner-authorized capture",
|
||||
},
|
||||
"streams": {
|
||||
"sensor.camera.right": stream,
|
||||
"sensor.lidar.registered-map-increment": stream,
|
||||
"sensor.pose": stream,
|
||||
"telemetry.pipeline": stream,
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"kind": "control-bridge",
|
||||
"monotonic_start_seconds": 100.0,
|
||||
"monotonic_end_seconds": 180.0,
|
||||
},
|
||||
{
|
||||
"kind": "new-route",
|
||||
"monotonic_start_seconds": 200.0,
|
||||
"monotonic_end_seconds": 700.0,
|
||||
},
|
||||
],
|
||||
"authority": {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_e43_accepts_complete_same_k1_capture_manifest() -> None:
|
||||
result = validate_future_capture_manifest(
|
||||
_capture_manifest(),
|
||||
protocol=_protocol(),
|
||||
)
|
||||
|
||||
assert result["accepted"] is True
|
||||
assert result["segments"] == ["control-bridge", "new-route"]
|
||||
assert result["blind_truth_labels_available"] is False
|
||||
|
||||
|
||||
def test_e43_rejects_missing_pipeline_stream() -> None:
|
||||
manifest = _capture_manifest()
|
||||
del manifest["streams"]["telemetry.pipeline"] # type: ignore[index]
|
||||
with pytest.raises(E43FutureCaptureProtocolError, match="stream set"):
|
||||
validate_future_capture_manifest(
|
||||
manifest,
|
||||
protocol=_protocol(),
|
||||
)
|
||||
|
||||
|
||||
def test_e43_grouped_partition_keeps_connected_evidence_together() -> None:
|
||||
candidates = [
|
||||
{
|
||||
"schema_version": E43_CANDIDATE_SCHEMA,
|
||||
"item_id": "a",
|
||||
"scene_id": "scene-1",
|
||||
"track_id": "track-1",
|
||||
"time_block_id": "time-1",
|
||||
"route_segment": "control-bridge",
|
||||
},
|
||||
{
|
||||
"schema_version": E43_CANDIDATE_SCHEMA,
|
||||
"item_id": "b",
|
||||
"scene_id": "scene-1",
|
||||
"track_id": "track-2",
|
||||
"time_block_id": "time-2",
|
||||
"route_segment": "control-bridge",
|
||||
},
|
||||
{
|
||||
"schema_version": E43_CANDIDATE_SCHEMA,
|
||||
"item_id": "c",
|
||||
"scene_id": "scene-2",
|
||||
"track_id": "track-3",
|
||||
"time_block_id": "time-3",
|
||||
"route_segment": "new-route",
|
||||
},
|
||||
{
|
||||
"schema_version": E43_CANDIDATE_SCHEMA,
|
||||
"item_id": "d",
|
||||
"scene_id": "scene-3",
|
||||
"track_id": None,
|
||||
"time_block_id": "time-4",
|
||||
"route_segment": "new-route",
|
||||
},
|
||||
]
|
||||
assignments = assign_grouped_future_partitions(
|
||||
candidates,
|
||||
seed="fixed-before-capture",
|
||||
blind_fraction=0.3,
|
||||
)
|
||||
|
||||
assert assignments["a"] == assignments["b"]
|
||||
assert set(assignments.values()) == {"blind-truth", "visible-diagnostic"}
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e44_data_amplification_audit import (
|
||||
E44DataAmplificationAuditError,
|
||||
analyze_data_amplification,
|
||||
)
|
||||
|
||||
|
||||
def _row(root: str, path: str, payload: bytes, kind: str) -> dict[str, object]:
|
||||
return {
|
||||
"root": root,
|
||||
"path": path,
|
||||
"byte_length": len(payload),
|
||||
"sha256": hashlib.sha256(payload).hexdigest(),
|
||||
"kind": kind,
|
||||
}
|
||||
|
||||
|
||||
def test_e44_measures_exact_cross_root_duplication() -> None:
|
||||
shared = b"camera-frame"
|
||||
report = analyze_data_amplification(
|
||||
[
|
||||
_row("e30", "frames/a.jpg", shared, "camera-image"),
|
||||
_row("e40", "frames/a.jpg", shared, "camera-image"),
|
||||
_row("e40", "report.json", b"{}", "metadata-or-report"),
|
||||
]
|
||||
)
|
||||
|
||||
assert report["logical_bytes"] == len(shared) * 2 + 2
|
||||
assert report["unique_content_bytes"] == len(shared) + 2
|
||||
assert report["duplicate_bytes"] == len(shared)
|
||||
assert report["duplicate_content_groups"] == 1
|
||||
assert report["largest_duplicate_groups"][0]["copies"] == 2
|
||||
assert report["largest_duplicate_groups"][0]["roots"] == ["e30", "e40"]
|
||||
assert report["roots"]["e40"]["duplicate_bytes_within_root"] == 0
|
||||
|
||||
|
||||
def test_e44_rejects_same_digest_with_inconsistent_lengths() -> None:
|
||||
digest = "a" * 64
|
||||
with pytest.raises(E44DataAmplificationAuditError, match="inconsistent"):
|
||||
analyze_data_amplification(
|
||||
[
|
||||
{
|
||||
"root": "e30",
|
||||
"path": "one.bin",
|
||||
"byte_length": 1,
|
||||
"sha256": digest,
|
||||
"kind": "other",
|
||||
},
|
||||
{
|
||||
"root": "e40",
|
||||
"path": "two.bin",
|
||||
"byte_length": 2,
|
||||
"sha256": digest,
|
||||
"kind": "other",
|
||||
},
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.pipeline_telemetry import (
|
||||
JsonlPipelineTelemetrySink,
|
||||
MqttPipelineTelemetrySink,
|
||||
PipelineTelemetryEmitter,
|
||||
PipelineTelemetryError,
|
||||
PipelineTelemetryIdentity,
|
||||
build_pipeline_telemetry_document,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
NORMALIZER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "deploy"
|
||||
/ "telemetry-plane"
|
||||
/ "normalizer"
|
||||
/ "normalizer.py"
|
||||
)
|
||||
|
||||
|
||||
def _normalizer() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"missioncore_pipeline_telemetry_normalizer",
|
||||
NORMALIZER_PATH,
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _identity() -> PipelineTelemetryIdentity:
|
||||
return PipelineTelemetryIdentity(
|
||||
contour_id="worker-006",
|
||||
agent_id="mission-core-worker",
|
||||
node_id="DESKTOP-OPJ8J04",
|
||||
lab_id="E41",
|
||||
run_id="run-001",
|
||||
request_id="request-001",
|
||||
source_id="ravnoves00",
|
||||
source_package_id="e41-predictor-package-example",
|
||||
method_id="frozen-e40-predictor/v1",
|
||||
frame_index=17,
|
||||
)
|
||||
|
||||
|
||||
def test_pipeline_document_is_accepted_without_losing_stage_identity() -> None:
|
||||
identity = _identity()
|
||||
document = build_pipeline_telemetry_document(
|
||||
identity=identity,
|
||||
stage_id="predict",
|
||||
state="completed",
|
||||
duration_ms=125.5,
|
||||
input_count=89,
|
||||
output_count=89,
|
||||
queue_wait_ms=2.25,
|
||||
observed_at_utc="2026-07-28T12:00:00Z",
|
||||
)
|
||||
|
||||
row = _normalizer()._normalize(
|
||||
identity.topic,
|
||||
json.dumps(document).encode(),
|
||||
)
|
||||
|
||||
assert row[1:5] == (
|
||||
"worker-006",
|
||||
"mission-core-worker",
|
||||
"DESKTOP-OPJ8J04",
|
||||
"pipeline",
|
||||
)
|
||||
assert row[9:13] == ("E41", "run-001", "request-001", 17)
|
||||
assert json.loads(row[6]) == {
|
||||
"lab_id": "E41",
|
||||
"method_id": "frozen-e40-predictor/v1",
|
||||
"request_id": "request-001",
|
||||
"run_id": "run-001",
|
||||
"source_id": "ravnoves00",
|
||||
"source_package_id": "e41-predictor-package-example",
|
||||
"stage_id": "predict",
|
||||
"stage_state": "completed",
|
||||
}
|
||||
stored = json.loads(row[13])
|
||||
assert stored["payload"]["event"]["duration_ms"] == 125.5
|
||||
assert stored["authority"]["commands_enabled"] is False
|
||||
|
||||
|
||||
def test_stage_context_emits_terminal_event_and_preserves_failure() -> None:
|
||||
published: list[tuple[str, dict[str, object]]] = []
|
||||
|
||||
class Sink:
|
||||
def publish(self, topic: str, payload: bytes) -> None:
|
||||
published.append((topic, json.loads(payload)))
|
||||
|
||||
ticks = iter((1_000_000_000, 1_125_500_000))
|
||||
emitter = PipelineTelemetryEmitter(
|
||||
identity=_identity(),
|
||||
sink=Sink(),
|
||||
clock_ns=lambda: next(ticks),
|
||||
)
|
||||
with emitter.stage("predict", input_count=89) as outcome:
|
||||
outcome.output_count = 89
|
||||
|
||||
assert [document["stage_state"] for _, document in published] == [
|
||||
"started",
|
||||
"completed",
|
||||
]
|
||||
assert published[1][1]["payload"]["event"]["duration_ms"] == 125.5
|
||||
assert published[1][1]["payload"]["event"]["output_count"] == 89
|
||||
|
||||
failure_ticks = iter((2_000_000_000, 2_001_000_000))
|
||||
failure_emitter = PipelineTelemetryEmitter(
|
||||
identity=_identity(),
|
||||
sink=Sink(),
|
||||
clock_ns=lambda: next(failure_ticks),
|
||||
)
|
||||
with (
|
||||
pytest.raises(ValueError, match="source failure"),
|
||||
failure_emitter.stage("evaluate"),
|
||||
):
|
||||
raise ValueError("source failure")
|
||||
assert published[-1][1]["stage_state"] == "failed"
|
||||
assert published[-1][1]["payload"]["event"]["error_type"] == "ValueError"
|
||||
assert "source failure" not in json.dumps(published[-1][1])
|
||||
|
||||
|
||||
def test_jsonl_sink_records_topic_bound_documents(tmp_path: Path) -> None:
|
||||
path = tmp_path / "telemetry" / "e41.jsonl"
|
||||
identity = _identity()
|
||||
sink = JsonlPipelineTelemetrySink(path)
|
||||
document = build_pipeline_telemetry_document(
|
||||
identity=identity,
|
||||
stage_id="package",
|
||||
state="completed",
|
||||
duration_ms=1.0,
|
||||
)
|
||||
|
||||
sink.publish(identity.topic, json.dumps(document).encode())
|
||||
|
||||
record = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert record["schema_version"] == "missioncore.pipeline-telemetry-record/v1"
|
||||
assert record["topic"] == identity.topic
|
||||
assert record["payload"]["stage_id"] == "package"
|
||||
assert path.stat().st_mode & 0o077 == 0
|
||||
|
||||
|
||||
def test_mqtt_sink_uses_qos_one_without_retention() -> None:
|
||||
calls: list[tuple[str, bytes, int, bool]] = []
|
||||
|
||||
class Client:
|
||||
def publish(
|
||||
self,
|
||||
topic: str,
|
||||
payload: bytes,
|
||||
qos: int,
|
||||
retain: bool,
|
||||
) -> SimpleNamespace:
|
||||
calls.append((topic, payload, qos, retain))
|
||||
return SimpleNamespace(rc=0)
|
||||
|
||||
MqttPipelineTelemetrySink(Client()).publish("topic", b"payload")
|
||||
|
||||
assert calls == [("topic", b"payload", 1, False)]
|
||||
|
||||
|
||||
def test_pipeline_telemetry_rejects_unsafe_identity_and_invalid_metrics() -> None:
|
||||
with pytest.raises(PipelineTelemetryError, match="contour_id"):
|
||||
PipelineTelemetryIdentity(
|
||||
contour_id="../worker",
|
||||
agent_id="agent",
|
||||
node_id="node",
|
||||
lab_id="E41",
|
||||
run_id="run",
|
||||
source_id="source",
|
||||
source_package_id="package",
|
||||
method_id="method",
|
||||
)
|
||||
with pytest.raises(PipelineTelemetryError, match="duration"):
|
||||
build_pipeline_telemetry_document(
|
||||
identity=_identity(),
|
||||
stage_id="predict",
|
||||
state="completed",
|
||||
)
|
||||
@@ -10,6 +10,7 @@ from fastapi import APIRouter, HTTPException
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import ValidationError
|
||||
|
||||
from k1link.web.compute_contour_api import default_compute_contour
|
||||
from k1link.web.system_telemetry_api import (
|
||||
EXPECTED_NODE_ID,
|
||||
WorkerConnectionProfile,
|
||||
@@ -17,6 +18,7 @@ from k1link.web.system_telemetry_api import (
|
||||
WorkerProfileStore,
|
||||
WorkerTelemetryService,
|
||||
_agent_raw_document,
|
||||
_profile_from_compute_contour,
|
||||
_ssh_arguments,
|
||||
build_system_telemetry_router,
|
||||
)
|
||||
@@ -272,6 +274,23 @@ def test_worker_telemetry_prefers_ndc_container_names_during_migration(
|
||||
assert triton["canonical_name"] == "ndc-mission-core-triton"
|
||||
|
||||
|
||||
def test_worker_telemetry_history_keeps_one_row_per_agent_observation(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
service = WorkerTelemetryService(
|
||||
WorkerProfileStore(tmp_path / "system"),
|
||||
lambda _: _probe(),
|
||||
cache_seconds=0,
|
||||
)
|
||||
|
||||
first = service.snapshot(10)
|
||||
second = service.snapshot(10)
|
||||
|
||||
assert len(first["history"]) == 1
|
||||
assert len(second["history"]) == 1
|
||||
assert second["history"][0]["observed_at_utc"] == "2026-07-27T12:00:00Z"
|
||||
|
||||
|
||||
def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None:
|
||||
document = _agent_raw_document(
|
||||
{
|
||||
@@ -340,6 +359,25 @@ def test_agent_metrics_are_mapped_to_the_existing_product_contract() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_compute_contour_maps_to_worker_identity_without_singleton_defaults() -> None:
|
||||
contour = default_compute_contour().model_copy(
|
||||
update={
|
||||
"contour_id": "field-worker",
|
||||
"agent_id": "field-agent",
|
||||
"display_name": "Field Worker",
|
||||
"expected_node_id": "FIELD-01",
|
||||
"address": "192.0.2.25",
|
||||
}
|
||||
)
|
||||
|
||||
profile = _profile_from_compute_contour(contour)
|
||||
|
||||
assert profile.profile_id == "field-worker"
|
||||
assert profile.display_name == "Field Worker"
|
||||
assert profile.expected_node_id == "FIELD-01"
|
||||
assert profile.address == "192.0.2.25"
|
||||
|
||||
|
||||
def test_profile_apply_fails_closed_on_wrong_node_and_keeps_old_profile(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
|
||||
@@ -122,6 +122,12 @@ def test_telegraf_upsert_merges_split_fields_in_one_series() -> None:
|
||||
assert "EXCLUDED.payload -> 'fields'" in NORMALIZER_SOURCE
|
||||
|
||||
|
||||
def test_normalizer_enforces_oss_compatible_bounded_retention() -> None:
|
||||
assert "DELETE FROM contour_telemetry_samples" in NORMALIZER_SOURCE
|
||||
assert "INTERVAL '30 days'" in NORMALIZER_SOURCE
|
||||
assert "RETENTION_INTERVAL_SECONDS" in NORMALIZER_SOURCE
|
||||
|
||||
|
||||
def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
|
||||
normalizer = _normalizer()
|
||||
with pytest.raises(ValueError, match="schema"):
|
||||
@@ -136,6 +142,99 @@ def test_normalizer_rejects_unschematized_pipeline_payload() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_normalizer_preserves_native_pipeline_stage_tags() -> None:
|
||||
normalizer = _normalizer()
|
||||
row = normalizer._normalize(
|
||||
"mission-core/v1/contours/worker-006/agents/mission-core-worker/pipeline",
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.agent-pipeline-telemetry/v1",
|
||||
"node_id": "DESKTOP-OPJ8J04",
|
||||
"observed_at_utc": "2026-07-28T12:00:00Z",
|
||||
"lab_id": "E41",
|
||||
"run_id": "run-001",
|
||||
"tags": {
|
||||
"node_id": "DESKTOP-OPJ8J04",
|
||||
"contour_id": "worker-006",
|
||||
"agent_id": "mission-core-worker",
|
||||
"lab_id": "E41",
|
||||
"run_id": "run-001",
|
||||
"source_id": "ravnoves00",
|
||||
"source_package_id": "e41-predictor-package-example",
|
||||
"method_id": "frozen-e40-predictor/v1",
|
||||
"stage_id": "predict",
|
||||
"stage_state": "completed",
|
||||
},
|
||||
"payload": {"state": "ready"},
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
assert json.loads(row[6]) == {
|
||||
"lab_id": "E41",
|
||||
"method_id": "frozen-e40-predictor/v1",
|
||||
"run_id": "run-001",
|
||||
"source_id": "ravnoves00",
|
||||
"source_package_id": "e41-predictor-package-example",
|
||||
"stage_id": "predict",
|
||||
"stage_state": "completed",
|
||||
}
|
||||
|
||||
|
||||
def test_normalizer_rejects_oversized_payload_and_series_identity() -> None:
|
||||
normalizer = _normalizer()
|
||||
with pytest.raises(ValueError, match="1 MiB"):
|
||||
normalizer._normalize(
|
||||
"mission-core/v1/contours/worker-006/agents/worker-006/host",
|
||||
b"{" + b"x" * normalizer.MAX_PAYLOAD_BYTES + b"}",
|
||||
)
|
||||
with pytest.raises(ValueError, match="too many tags"):
|
||||
normalizer._normalize(
|
||||
"mission-core/v1/contours/worker-006/agents/worker-006/host",
|
||||
json.dumps(
|
||||
{
|
||||
"name": "cpu",
|
||||
"tags": {
|
||||
"node_id": "DESKTOP-OPJ8J04",
|
||||
**{
|
||||
f"tag-{index}": str(index)
|
||||
for index in range(normalizer.MAX_TAGS + 1)
|
||||
},
|
||||
},
|
||||
"fields": {"usage_active": 12.5},
|
||||
"timestamp": 1_785_179_600,
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
def test_normalizer_removes_unneeded_docker_labels_and_host_bind_paths() -> None:
|
||||
normalizer = _normalizer()
|
||||
row = normalizer._normalize(
|
||||
"mission-core/v1/contours/worker-006/agents/worker-006/host",
|
||||
json.dumps(
|
||||
{
|
||||
"name": "docker_container_cpu",
|
||||
"tags": {
|
||||
"node_id": "DESKTOP-OPJ8J04",
|
||||
"container_name": "ndc-mission-core-triton",
|
||||
"desktop.docker.io/binds/0/Source": "C:\\private\\model",
|
||||
"com.nvidia.cuda.version": "12.8",
|
||||
},
|
||||
"fields": {"usage_percent": 10.0},
|
||||
"timestamp": 1_785_179_600,
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
assert row[6] == '{"container_name":"ndc-mission-core-triton"}'
|
||||
stored = json.loads(row[13])
|
||||
assert stored["tags"] == {
|
||||
"node_id": "DESKTOP-OPJ8J04",
|
||||
"container_name": "ndc-mission-core-triton",
|
||||
}
|
||||
|
||||
|
||||
def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
|
||||
document = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8"))
|
||||
|
||||
@@ -146,6 +245,7 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
|
||||
for service in services.values()
|
||||
} == {
|
||||
"ndc-mission-core-mqtt-broker",
|
||||
"ndc-mission-core-telemetry-bootstrap",
|
||||
"ndc-mission-core-telemetry-normalizer",
|
||||
"ndc-mission-core-telemetry-timescaledb",
|
||||
}
|
||||
@@ -154,6 +254,8 @@ def test_telemetry_plane_uses_the_ndc_docker_namespace() -> None:
|
||||
assert service["labels"]["com.nodedc.product"] == "mission-core"
|
||||
assert service["labels"]["com.nodedc.stack"] == "ndc-mission-core-telemetry"
|
||||
|
||||
assert services["broker"]["cap_drop"] == ["ALL"]
|
||||
assert set(services["broker"]["cap_add"]) == {"CHOWN", "SETGID", "SETUID"}
|
||||
assert document["networks"]["default"]["name"] == "ndc-mission-core-telemetry"
|
||||
assert {
|
||||
volume["name"]
|
||||
|
||||
@@ -45,10 +45,11 @@ def test_initialize_environment_generates_private_unique_secrets(
|
||||
assert values["MISSIONCORE_MQTT_BIND_ADDRESS"] == "192.0.2.15"
|
||||
secrets = {
|
||||
values["MISSIONCORE_DB_PASSWORD"],
|
||||
values["MISSIONCORE_DB_INGEST_PASSWORD"],
|
||||
values["MISSIONCORE_MQTT_INGEST_PASSWORD"],
|
||||
values["MISSIONCORE_MQTT_WORKER_006_PASSWORD"],
|
||||
}
|
||||
assert len(secrets) == 3
|
||||
assert len(secrets) == 4
|
||||
assert all(len(secret) >= 40 for secret in secrets)
|
||||
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
|
||||
|
||||
@@ -65,3 +66,61 @@ def test_initialize_environment_refuses_to_replace_credentials(
|
||||
prepare._initialize_environment("127.0.0.1")
|
||||
|
||||
assert env_path.read_text(encoding="utf-8") == "existing=true\n"
|
||||
|
||||
|
||||
def test_environment_migration_adds_only_new_private_values(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
env_path = tmp_path / ".env"
|
||||
env_path.write_text(
|
||||
"MISSIONCORE_DB_PASSWORD=keep-me\n"
|
||||
"MISSIONCORE_MQTT_WORKER_006_USER=worker-006\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(prepare, "ENV_PATH", env_path)
|
||||
|
||||
prepare._migrate_environment()
|
||||
first = env_path.read_text(encoding="utf-8")
|
||||
prepare._migrate_environment()
|
||||
|
||||
assert "MISSIONCORE_DB_PASSWORD=keep-me" in first
|
||||
assert "MISSIONCORE_DB_INGEST_PASSWORD=" in first
|
||||
assert "MISSIONCORE_MQTT_WORKER_006_CONTOUR=worker-006" in first
|
||||
assert env_path.read_text(encoding="utf-8") == first
|
||||
assert stat.S_IMODE(env_path.stat().st_mode) == 0o600
|
||||
|
||||
|
||||
def test_existing_password_file_is_updated_without_recreation(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
password_path = tmp_path / "passwords"
|
||||
password_path.write_text("existing", encoding="utf-8")
|
||||
calls: list[tuple[str, bool]] = []
|
||||
|
||||
def capture(
|
||||
path: Path,
|
||||
username: str,
|
||||
password: str,
|
||||
*,
|
||||
create: bool,
|
||||
) -> None:
|
||||
assert path == password_path
|
||||
assert password
|
||||
calls.append((username, create))
|
||||
|
||||
monkeypatch.setattr(prepare, "_password_entry", capture)
|
||||
|
||||
prepare._prepare_password_entries(
|
||||
password_path,
|
||||
"missioncore-ingest",
|
||||
"ingest-secret",
|
||||
"worker-006",
|
||||
"worker-secret",
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("missioncore-ingest", False),
|
||||
("worker-006", False),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user