SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(lab): measure RAVNOVES00 R1 quality baseline

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-28 03:42:32 +03:00
parent e35f81ac34
commit c018f2b074
20 changed files with 2401 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_advanced_laboratory_api.py
+108 -3
View File
@@ -26,7 +26,7 @@ def _endpoint(router: APIRouter, path: str) -> object:
def test_advanced_catalogs_are_empty_when_not_configured() -> None:
router = build_advanced_laboratory_router()
for name in ("e31", "e32", "e33", "e34", "e35", "e37"):
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38"):
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog == {
@@ -48,7 +48,8 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
e34 = tmp_path / "e34"
e35 = tmp_path / "e35"
e37 = tmp_path / "e37"
for root in (e31, e32, e33, e34, e35, e37):
e38 = tmp_path / "e38"
for root in (e31, e32, e33, e34, e35, e37, e38):
root.mkdir()
(e31 / f"e31-source-qualification-{'1' * 64}").mkdir()
(e32 / f"e32-track-geometry-{'2' * 64}").mkdir()
@@ -56,6 +57,7 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
(e34 / f"e34-temporal-occupied-{'4' * 64}").mkdir()
(e35 / f"e35-degradation-recovery-{'5' * 64}").mkdir()
(e37 / f"e37-ravnoves-acceptance-{'7' * 64}").mkdir()
(e38 / f"e38-perception-baseline-{'8' * 64}").mkdir()
router = build_advanced_laboratory_router(
e31_root_provider=lambda: e31,
e32_root_provider=lambda: e32,
@@ -63,9 +65,10 @@ def test_advanced_catalogs_fail_closed_on_incomplete_results(
e34_root_provider=lambda: e34,
e35_root_provider=lambda: e35,
e37_root_provider=lambda: e37,
e38_root_provider=lambda: e38,
)
for name in ("e31", "e32", "e33", "e34", "e35", "e37"):
for name in ("e31", "e32", "e33", "e34", "e35", "e37", "e38"):
route = _endpoint(router, f"/api/v1/laboratory/{name}/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog["configured"] is True
@@ -422,3 +425,105 @@ def test_e37_catalog_projects_the_frozen_r0_contract(
assert item["quality_target_evaluated"] is False
assert item["authority"] == authority
assert item["access"] == "read-only"
def test_e38_catalog_projects_failed_dimensions_without_hiding_result(
tmp_path: Path,
monkeypatch: MonkeyPatch,
) -> None:
result_id = f"e38-perception-baseline-{'8' * 64}"
root = tmp_path / "e38"
candidate = root / result_id
candidate.mkdir(parents=True)
(candidate / "manifest.json").write_text("{}", encoding="utf-8")
authority = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
dimension = {
"correct": 120,
"incorrect": 26,
"total": 146,
"accuracy": 0.821918,
"target": 0.9,
"passed": False,
"confusion": [],
"by_stratum": {},
}
result = SimpleNamespace(
result_id=result_id,
manifest={
"created_at_utc": "2026-07-28T02:50:00Z",
"identity": {
"source": {
"session_id": "20260720T065719Z_viewer_live",
"display_name": "RAVNOVES00",
},
"profile": {
"profile_id": "e38-ravnoves00-r1-development-cart/v1",
},
"execution": {
"worker_node": "DESKTOP-OPJ8J04",
},
},
},
report={
"status": "measured-r1-source-scoped-baseline",
"metrics": {
"development_items": 340,
"validation_items": 146,
"terminal_outcomes": 146,
"accounting_fraction": 1.0,
"false_free_claims": 0,
"high_severity_failures": 14,
"dimensions": {
"presence": dimension,
"geometry_association": dimension,
"freshness": {
**dimension,
"correct": 139,
"incorrect": 7,
"accuracy": 0.952055,
"passed": True,
},
},
},
"quality_gate": {
"passed": False,
"blocking_checks": [
"presence_target_reached",
"geometry_association_target_reached",
],
},
"decision": {
"r1_baseline_measured": True,
"accepted_for_release": False,
},
"limitations": ["source-scoped"],
"authority": authority,
},
)
def fake_read(
root_text: str,
signature: tuple[int, ...],
) -> SimpleNamespace:
assert root_text == str(candidate.resolve())
assert signature
return result
monkeypatch.setattr(advanced_api, "_read_e38_cached", fake_read)
router = build_advanced_laboratory_router(
e38_root_provider=lambda: root,
)
route = _endpoint(router, "/api/v1/laboratory/e38/results")
catalog = route(limit=1) # type: ignore[operator]
assert catalog["candidate_total"] == 1
assert catalog["invalid_total"] == 0
item = catalog["items"][0]
assert item["quality_gate_passed"] is False
assert item["metrics"]["dimensions"]["freshness"]["passed"] is True
assert item["metrics"]["dimensions"]["presence"]["passed"] is False
assert item["authority"] == authority
assert item["access"] == "read-only"
tests/test_e38_perception_baseline.py
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from k1link.compute.e38_perception_baseline import (
_dimension_metrics,
_predict_tree,
_train_tree,
)
def test_e38_cart_is_deterministic_and_respects_minimum_leaf() -> None:
rows = [
({"score": 0.1, "support": 0.0}, "background"),
({"score": 0.2, "support": 1.0}, "background"),
({"score": 0.7, "support": 3.0}, "object"),
({"score": 0.8, "support": 4.0}, "object"),
({"score": 0.9, "support": 5.0}, "object"),
({"score": 1.0, "support": 6.0}, "object"),
]
first = _train_tree(
rows,
feature_names=["score", "support"],
max_depth=3,
min_leaf=2,
)
second = _train_tree(
rows,
feature_names=["score", "support"],
max_depth=3,
min_leaf=2,
)
assert first == second
assert _predict_tree(first, {"score": 0.15, "support": 1.0}) == "background"
assert _predict_tree(first, {"score": 0.85, "support": 4.0}) == "object"
def test_e38_dimension_metrics_do_not_blend_strata() -> None:
rows = [
{
"source_stratum": "agree",
"prediction": {"presence": "object-present"},
"reference": {"presence": "object-present"},
},
{
"source_stratum": "agree",
"prediction": {"presence": "object-present"},
"reference": {"presence": "background-or-noise"},
},
{
"source_stratum": "geometry-only",
"prediction": {"presence": "occupied-environment"},
"reference": {"presence": "occupied-environment"},
},
]
metrics = _dimension_metrics(rows, dimension="presence", target=0.9)
assert metrics["correct"] == 2
assert metrics["incorrect"] == 1
assert metrics["accuracy"] == 0.666667
assert metrics["passed"] is False
assert metrics["by_stratum"]["agree"]["accuracy"] == 0.5
assert metrics["by_stratum"]["geometry-only"]["accuracy"] == 1.0
tests/test_e38_worker_package.py
+86
View File
@@ -0,0 +1,86 @@
from __future__ import annotations
import importlib.util
import subprocess
import sys
from pathlib import Path
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e38_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e38_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e38_package_is_minimal_content_addressed_projection(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
acceptance = (
repository
/ ".runtime"
/ "compute-experiments"
/ "e37"
/ "results"
/ (
"e37-ravnoves-acceptance-"
"01b1efd586f747341c712d82f0907b39436a6f91ae92b1dfae987eca05fd8344"
)
)
materialization = (
repository
/ ".runtime"
/ "compute-experiments"
/ "e30"
/ "materializations"
/ (
"e30-materialization-"
"841af926d8d28ab93538c46d8f31278a2234c4d1c12c7dc4dc296b249d59735a"
)
)
package = module.build_e38_worker_package(
repository_root=repository,
acceptance_root=acceptance,
materialization_root=materialization,
profile_path=(
repository
/ "experiments"
/ "perception"
/ "e38_ravnoves00_r1_baseline_profile.json"
),
output_root=tmp_path,
)
manifest = module.validate_e38_worker_package(package)
assert package.name == f"e38-worker-package-{manifest['identity_sha256']}"
assert manifest["identity"]["classification"] == (
"immutable-ravnoves00-r1-worker-input"
)
assert len(manifest["artifacts"]) == 13
completed = subprocess.run(
[
sys.executable,
str(package / "runtime" / "run_e38_perception_baseline.py"),
"--package",
str(package),
"--output-root",
str(tmp_path / "results"),
],
env={
"PYTHONPATH": str(package / "runtime"),
"PYTHONDONTWRITEBYTECODE": "1",
"E38_WORKER_NODE": "TEST-WORKER-006",
},
check=False,
capture_output=True,
text=True,
)
assert completed.returncode == 0, completed.stderr
assert '"quality_gate_passed": false' in completed.stdout