feat(lab): publish M4.8S fixed-class detector replay
This commit is contained in:
@@ -127,7 +127,7 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
repository_root / "config" / "laboratories"
|
||||
)
|
||||
|
||||
assert len(registry.definitions) == 36
|
||||
assert len(registry.definitions) == 37
|
||||
assert {item.work_id for item in registry.definitions} >= {
|
||||
"e31-source-binding",
|
||||
"e46j-raw-fisheye-realtime",
|
||||
@@ -141,6 +141,7 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
"m47-reference-graph-shadow",
|
||||
"m48-object-centric-quality",
|
||||
"m48-small-static-passage-regression",
|
||||
"m48s-fixed-class-detector",
|
||||
}
|
||||
m48 = next(
|
||||
item for item in registry.definitions
|
||||
|
||||
@@ -98,6 +98,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"e35-degradation-recovery",
|
||||
"e46j-raw-fisheye-realtime",
|
||||
"e47-semantic-slam-shadow",
|
||||
"m48s-fixed-class-detector",
|
||||
}
|
||||
by_work_id = {row.work_id: row for row in execution.definitions}
|
||||
assert by_work_id["m48-small-static-passage-regression"].evidence_contract == (
|
||||
@@ -108,10 +109,12 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
)
|
||||
assert by_work_id["e47-semantic-slam-shadow"].lifecycle == "experimental"
|
||||
assert by_work_id["e47-semantic-slam-shadow"].isolation == "bounded-adapter"
|
||||
assert by_work_id["m48s-fixed-class-detector"].lifecycle == "experimental"
|
||||
assert by_work_id["m48s-fixed-class-detector"].isolation == "bounded-adapter"
|
||||
assert all(
|
||||
row.lifecycle == "canonical"
|
||||
for row in execution.definitions
|
||||
if row.work_id != "e47-semantic-slam-shadow"
|
||||
if row.work_id not in {"e47-semantic-slam-shadow", "m48s-fixed-class-detector"}
|
||||
)
|
||||
assert len(execution.definitions) + len(execution.legacy_work_ids) == len(
|
||||
evidence.definitions
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
root / "config" / "laboratory-value-review.json"
|
||||
)
|
||||
|
||||
assert len(registry.entries) == 35
|
||||
assert len(registry.entries) == 36
|
||||
assert {entry.catalog_id for entry in registry.entries} >= {
|
||||
"e28-local-surface",
|
||||
"e46d-temporal-failure-audit",
|
||||
@@ -88,4 +88,5 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
"l31-pointpillars-ravnoves",
|
||||
"l34f-adjudicated-reference",
|
||||
"m4-replay-threat",
|
||||
"m48s-fixed-class-detector",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.laboratory import LaboratoryEvidenceRegistry
|
||||
from k1link.laboratory.evidence_report import verify_laboratory_evidence_result
|
||||
from k1link.laboratory.m48s_fixed_class_detector_lab import (
|
||||
FRAME_IDS,
|
||||
LAB_SCHEMA,
|
||||
M48SFixedClassDetectorLabError,
|
||||
build_m48s_fixed_class_detector_lab,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def test_m48s_lab_seals_visual_comparison_and_load_evidence(tmp_path: Path) -> None:
|
||||
result = build_m48s_fixed_class_detector_lab(
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
manifest = result.manifest
|
||||
identity = manifest["identity"]
|
||||
identity_digest = hashlib.sha256(
|
||||
json.dumps(
|
||||
identity,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
assert manifest["schema_version"] == LAB_SCHEMA
|
||||
assert manifest["result_id"] == result.result_id
|
||||
assert result.result_id.endswith(identity_digest)
|
||||
assert manifest["identity_sha256"] == identity_digest
|
||||
assert len(manifest["artifacts"]) == 31
|
||||
assert manifest["method"]["completeness"] == "complete"
|
||||
assert manifest["bounded_question_accepted"] is True
|
||||
assert manifest["ground_truth"] is False
|
||||
assert manifest["decision"] == {
|
||||
"bounded_question_accepted": True,
|
||||
"selected_candidate": "rf-detr",
|
||||
"ready_for_reference_graph_shadow": True,
|
||||
"integrated_world_state_gate_evaluated": True,
|
||||
"integrated_world_state_gate_passed": True,
|
||||
"full_replay_visual_published": True,
|
||||
"detector_replacement_authorized": False,
|
||||
"production_accepted": False,
|
||||
}
|
||||
assert manifest["authority"]["navigation_or_safety_accepted"] is False
|
||||
assert manifest["metrics"]["detector_load"]["source_frames_consumed"] == 18_008
|
||||
assert manifest["metrics"]["detector_load"]["source_frame_replacements"] == 0
|
||||
assert manifest["metrics"]["detector_load"]["effective_consumed_fps"] >= 9.5
|
||||
assert manifest["metrics"]["detector_load"]["completion_age_p95_ms"] <= 175.0
|
||||
integrated = manifest["metrics"]["integrated_world_state"]
|
||||
assert integrated["source_frames_admitted"] == 4_489
|
||||
assert integrated["delivered_world_states"] == 4_481
|
||||
assert integrated["superseded_frames"] == 8
|
||||
assert integrated["effective_world_state_fps"] >= 9.5
|
||||
assert integrated["world_state_completion_age_p95_ms"] <= 175.0
|
||||
assert max(integrated["queue_high_watermarks"].values()) <= 2
|
||||
assert integrated["additional_inference_passes"] == 0
|
||||
assert integrated["failures"] == 0
|
||||
|
||||
catalog = json.loads((result.result_root / "catalog.json").read_text("utf-8"))
|
||||
assert catalog["frame_count"] == len(FRAME_IDS) == 11
|
||||
frame = json.loads((result.result_root / "frame-000253.json").read_text("utf-8"))
|
||||
assert not any(item["label"] == "dog" for item in frame["detections"]["yolox"])
|
||||
assert not any(item["label"] == "dog" for item in frame["detections"]["dfine"])
|
||||
assert any(item["label"] == "skateboard" for item in frame["detections"]["dfine"])
|
||||
assert [item["score"] for item in frame["detections"]["rf-detr"] if item["label"] == "dog"] == [
|
||||
0.741674
|
||||
]
|
||||
|
||||
registry = LaboratoryEvidenceRegistry.from_directory(
|
||||
REPOSITORY_ROOT / "config" / "laboratories"
|
||||
)
|
||||
definition = next(
|
||||
item for item in registry.definitions if item.work_id == "m48s-fixed-class-detector"
|
||||
)
|
||||
proof = verify_laboratory_evidence_result(definition, result.result_root)
|
||||
assert proof["result_id"] == result.result_id
|
||||
assert proof["artifact_count"] == 31
|
||||
|
||||
with pytest.raises(M48SFixedClassDetectorLabError, match="already exists"):
|
||||
build_m48s_fixed_class_detector_lab(
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.laboratory.m48s_fixed_class_detector_lab import (
|
||||
build_m48s_fixed_class_detector_lab,
|
||||
)
|
||||
from k1link.web.m48s_fixed_class_detector_lab_api import (
|
||||
build_m48s_fixed_class_detector_lab_router,
|
||||
)
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _fixture(tmp_path: Path) -> tuple[TestClient, Path, str]:
|
||||
root = tmp_path / "results"
|
||||
result = build_m48s_fixed_class_detector_lab(
|
||||
repository_root=REPOSITORY_ROOT,
|
||||
output_root=root,
|
||||
)
|
||||
app = FastAPI()
|
||||
app.include_router(
|
||||
build_m48s_fixed_class_detector_lab_router(
|
||||
root_provider=lambda: root,
|
||||
repository_root_provider=lambda: REPOSITORY_ROOT,
|
||||
)
|
||||
)
|
||||
return TestClient(app), result.result_root, result.result_id
|
||||
|
||||
|
||||
def test_m48s_lab_api_projects_verified_result_frame_and_camera(tmp_path: Path) -> None:
|
||||
client, result_root, result_id = _fixture(tmp_path)
|
||||
|
||||
catalog = client.get("/api/v1/laboratory/m48s/fixed-class-detector/results")
|
||||
assert catalog.status_code == 200
|
||||
assert catalog.json()["items"][0]["result_id"] == result_id
|
||||
assert catalog.json()["invalid_total"] == 0
|
||||
|
||||
result = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}")
|
||||
assert result.status_code == 200
|
||||
assert result.json()["decision"]["selected_candidate"] == "rf-detr"
|
||||
assert result.json()["decision"]["integrated_world_state_gate_passed"] is True
|
||||
assert (
|
||||
result.json()["metrics"]["integrated_world_state"]["world_state_completion_age_p95_ms"]
|
||||
== 74.733648
|
||||
)
|
||||
assert result.json()["ground_truth"] is False
|
||||
assert len(result.json()["frames"]) == 11
|
||||
|
||||
timeline = client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline"
|
||||
)
|
||||
assert timeline.status_code == 200
|
||||
assert timeline.json()["frame_count"] == 4_489
|
||||
assert timeline.json()["world_state_frame_count"] == 4_481
|
||||
assert timeline.json()["superseded_frame_count"] == 8
|
||||
assert (
|
||||
timeline.json()["camera_point_delivery"]
|
||||
== "factory-kb4-projected-current-increment"
|
||||
)
|
||||
chunk = client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/timeline/chunk",
|
||||
params={"start": 253, "count": 1},
|
||||
)
|
||||
assert chunk.status_code == 200
|
||||
replay_frame = chunk.json()["frames"][0]
|
||||
assert replay_frame["world_state_available"] is True
|
||||
assert replay_frame["camera_projection"] == "factory-kb4-exact"
|
||||
assert replay_frame["camera_projected_sample_count"] > 0
|
||||
assert any(
|
||||
item["semantic_hint"] == "dog" for item in replay_frame["camera_proposals"]
|
||||
)
|
||||
|
||||
frame = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/frames/000253")
|
||||
assert frame.status_code == 200
|
||||
assert any(item["label"] == "dog" for item in frame.json()["detections"]["rf-detr"])
|
||||
assert frame.json()["access"] == "read-only"
|
||||
|
||||
camera = client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/frames/000253/camera"
|
||||
)
|
||||
assert camera.status_code == 200
|
||||
assert camera.headers["content-type"] == "image/jpeg"
|
||||
assert camera.content == (result_root / "frames/frame-000253.jpg").read_bytes()
|
||||
|
||||
assert (
|
||||
client.get(
|
||||
f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}/frames/not-a-frame"
|
||||
).status_code
|
||||
== 404
|
||||
)
|
||||
assert (
|
||||
client.get("/api/v1/laboratory/m48s/fixed-class-detector/not-a-result").status_code == 404
|
||||
)
|
||||
|
||||
|
||||
def test_m48s_lab_api_fails_closed_after_artifact_tamper(tmp_path: Path) -> None:
|
||||
client, result_root, result_id = _fixture(tmp_path)
|
||||
(result_root / "frame-000253.json").write_text("{}\n", encoding="utf-8")
|
||||
|
||||
response = client.get(f"/api/v1/laboratory/m48s/fixed-class-detector/{result_id}")
|
||||
assert response.status_code == 404
|
||||
|
||||
catalog = client.get("/api/v1/laboratory/m48s/fixed-class-detector/results")
|
||||
assert catalog.json()["items"] == []
|
||||
assert catalog.json()["invalid_total"] == 1
|
||||
Reference in New Issue
Block a user