feat(perception): add recorded replay maturation labs

This commit is contained in:
DCCONSTRUCTIONS
2026-08-05 07:47:15 +03:00
parent c1b0f6f8a3
commit 3982256f08
101 changed files with 24009 additions and 4 deletions
@@ -0,0 +1,325 @@
from __future__ import annotations
import copy
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi import APIRouter
from fastapi.routing import APIRoute
import k1link.compute.e46a_ai_engineering_preannotation as compute
from k1link.web.e46a_ai_engineering_preannotation_api import (
build_e46a_ai_engineering_preannotation_router,
)
from k1link.web.l34_annotation_api import (
L34AnnotationCreateRequest,
L34AnnotationFrameRequest,
L34AnnotationObjectRequest,
L34AnnotationSaveRequest,
)
def _endpoint(router: APIRouter, path: str, method: str = "GET") -> object:
for route in router.routes:
if (
isinstance(route, APIRoute)
and route.path == path
and route.methods is not None
and method in route.methods
):
return route.endpoint
raise AssertionError(f"{method} {path} route is missing")
def _build_fixture(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> dict[str, object]:
e46_id = f"e46-detector-truth-island-{'a' * 64}"
e46_root = tmp_path / "e46" / e46_id
e46_root.mkdir(parents=True)
(e46_root / "manifest.json").write_text("{}\n", encoding="utf-8")
references = []
cases = []
for sequence in range(1, 33):
source_sha = f"{sequence:064x}"
references.append(
{
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence * 100,
"group_id": f"group-{sequence:02d}",
"sha256": source_sha,
}
)
objects = [
{
"object_id": f"object-{sequence:02d}-car",
"category": "car",
"proposed_label": None,
"origin": "self_review_seed",
"box_xyxy": [10.0, 20.0, 100.0, 120.0],
"occluded": False,
"truncated": False,
}
]
if sequence in {5, 29}:
objects.append(
{
"object_id": f"object-{sequence:02d}-custom",
"category": "unmapped",
"proposed_label": (
"Детская коляска" if sequence == 5 else "Ноутбук"
),
"origin": "self_review_seed",
"box_xyxy": [120.0, 140.0, 280.0, 320.0],
"occluded": False,
"truncated": sequence == 29,
}
)
cases.append(
{
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence * 100,
"group_id": f"group-{sequence:02d}",
"session_seconds": float(sequence),
"source_image_sha256": source_sha,
"references": objects,
}
)
(e46_root / "image-references.jsonl").write_text(
"".join(json.dumps(row) + "\n" for row in references),
encoding="utf-8",
)
l34f_id = f"l34f-adjudicated-reference-{'b' * 64}"
l34f_root = tmp_path / "l34f" / l34f_id
l34f_root.mkdir(parents=True)
(l34f_root / "manifest.json").write_text("{}\n", encoding="utf-8")
e46 = SimpleNamespace(
result_id=e46_id,
result_root=e46_root,
manifest={},
report={},
)
l34f = {
"result_id": l34f_id,
"result_root": l34f_root,
"manifest": {},
"report": {},
"cases": tuple(cases),
}
monkeypatch.setattr(compute, "read_e46_detector_truth_island", lambda _: e46)
monkeypatch.setattr(compute, "read_l34f_adjudicated_reference", lambda _: l34f)
return compute.build_e46a_ai_engineering_preannotation(
e46_root=e46_root,
l34f_root=l34f_root,
output_root=tmp_path / "results",
)
def test_e46a_freezes_ai_preannotation_without_truth_authority(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
result = _build_fixture(tmp_path, monkeypatch)
assert result["report"]["metrics"]["frame_count"] == 32
assert result["report"]["metrics"]["object_count"] == 34
assert result["report"]["metrics"]["custom_class_relabel_count"] == 2
assert result["report"]["metrics"]["independent_review_submission_count"] == 0
assert result["report"]["decision"]["e48_truth_seal_open"] is False
assert result["manifest"]["authority"]["independent_truth"] is False
custom = {
item["category"]
for case in result["cases"]
for item in case["objects"]
if item["category"] in {"stroller", "laptop"}
}
assert custom == {"stroller", "laptop"}
assert all(
item["category"] != "unmapped"
for case in result["cases"]
for item in case["objects"]
)
def test_e46a_object_level_audit_deletes_snaps_and_relabels(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
result = _build_fixture(tmp_path, monkeypatch)
cases = copy.deepcopy(list(result["cases"]))
rows = tuple(
{
"candidate_id": "geometry-candidate",
"truth_island_sequence": sequence,
"source_image_sha256": cases[sequence - 1][
"source_image_sha256"
],
"predictions": [
{
"category": "car",
"score": 0.95,
"box_xyxy": [12.0, 22.0, 98.0, 118.0],
}
],
}
for sequence in range(1, 33)
)
profile = compute.E46AVisualAuditProfile(
profile_id="fixture-object-qa/v1",
geometry_candidate_id="geometry-candidate",
candidate_nms_iou=0.3,
maximum_match_cost=1.3,
expected_source_object_count=34,
expected_final_object_count=33,
expected_geometry_snapped_count=31,
delete_object_ids=("object-01-car",),
category_overrides=(("object-02-car", "heavy_vehicle"),),
)
audit = compute._apply_visual_audit( # noqa: SLF001
cases=cases,
prediction_rows=rows,
profile=profile,
)
assert audit["deleted_false_box_count"] == 1
assert audit["geometry_snapped_object_count"] == 31
assert audit["source_geometry_retained_object_count"] == 2
assert cases[0]["objects"] == []
assert cases[0]["hard_negative"] is True
assert cases[1]["objects"][0]["category"] == "heavy_vehicle"
assert cases[1]["objects"][0]["box_xyxy"] == [12.0, 22.0, 98.0, 118.0]
def test_e46a_api_projects_visual_cases_without_truth_escalation(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
result = _build_fixture(tmp_path, monkeypatch)
router = build_e46a_ai_engineering_preannotation_router(
root_provider=lambda: tmp_path / "results"
)
catalog = _endpoint(router, "/api/v1/laboratory/e46a/results")(limit=1) # type: ignore[operator]
assert catalog["items"][0]["result_id"] == result["result_id"]
assert catalog["items"][0]["ground_truth"] is False
case = _endpoint(
router,
"/api/v1/laboratory/e46a/results/{result_id}/cases/{sequence}",
)(result_id=result["result_id"], sequence=5) # type: ignore[operator]
assert case["independent_review"] is False
assert case["ground_truth"] is False
assert any(item["category"] == "stroller" for item in case["objects"])
assert case["camera_url"].endswith("/annotation-source/frames/5/camera")
def test_e46a_correction_session_starts_from_editable_ai_seed(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
result = _build_fixture(tmp_path, monkeypatch)
router = build_e46a_ai_engineering_preannotation_router(
root_provider=lambda: tmp_path / "results",
annotation_root_provider=lambda: tmp_path / "annotations",
)
result_id = str(result["result_id"])
source = _endpoint(
router,
"/api/v1/laboratory/e46a/results/{result_id}/annotation-source",
)(result_id=result_id) # type: ignore[operator]
assert source["frame_count"] == 32
assert source["contract"]["contract_id"] == (
"e46a-ai-engineering-correction/v1"
)
assert source["candidate_identity_included"] is True
assert source["candidate_predictions_included"] is False
assert source["prelabels_included"] is False
seed = _endpoint(
router,
(
"/api/v1/laboratory/e46a/results/{result_id}"
"/annotation-seed/frames/{sequence}"
),
)(result_id=result_id, sequence=5) # type: ignore[operator]
custom = next(
item for item in seed["objects"] if item["category"] == "unmapped"
)
assert custom["proposed_label"] == "Коляска"
assert custom["origin"] == "frozen_candidate_seed"
create = _endpoint(
router,
"/api/v1/laboratory/e46a/results/{result_id}/annotation-sessions",
"POST",
)
created = create( # type: ignore[operator]
result_id=result_id,
request=L34AnnotationCreateRequest(idempotency_key="e46a-correction-1"),
)
assert created["contract_id"] == "e46a-ai-engineering-correction/v1"
assert created["revision"] == 0
assert created["frames"] == []
save = _endpoint(
router,
(
"/api/v1/laboratory/e46a/results/{result_id}"
"/annotation-sessions/{session_id}"
),
"PUT",
)
saved = save( # type: ignore[operator]
result_id=result_id,
session_id=created["session_id"],
request=L34AnnotationSaveRequest(
expected_revision=0,
idempotency_key="e46a-save-1",
title="E46A human correction",
assistance_mode="frozen-candidate-seeded",
frames=[
L34AnnotationFrameRequest(
truth_island_sequence=5,
reviewed=True,
hard_negative=False,
objects=[
L34AnnotationObjectRequest(
object_id=str(custom["object_id"]),
category="unmapped",
proposed_label="Коляска",
origin="frozen_candidate_seed",
box_xyxy=[130.0, 150.0, 300.0, 340.0],
occluded=False,
truncated=False,
)
],
)
],
),
)
assert saved["revision"] == 1
assert saved["frames"][0]["objects"][0]["box_xyxy"] == [
130.0,
150.0,
300.0,
340.0,
]
assert saved["authority"]["ground_truth"] is False
def test_e46a_reader_rejects_tampered_cases(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
result = _build_fixture(tmp_path, monkeypatch)
cases_path = Path(result["result_root"]) / compute.E46A_CASES_NAME
cases_path.write_text(cases_path.read_text(encoding="utf-8") + "{}\n", encoding="utf-8")
with pytest.raises(compute.E46AAiEngineeringPreannotationError):
compute.read_e46a_ai_engineering_preannotation(Path(result["result_root"]))
+83
View File
@@ -0,0 +1,83 @@
from __future__ import annotations
from k1link.compute.e46d_temporal_failure_audit import analyze_temporal_frames
def _object(
track_id: int,
*,
box: list[float] | None = None,
camera_current: bool = True,
world_track_id: int | None = 240001,
motion_state: str = "static",
) -> dict[str, object]:
return {
"bbox_xyxy": box or [100.0, 100.0, 180.0, 220.0],
"category": "car",
"route_track_id": track_id,
"world_track_id": world_track_id,
"motion_state": motion_state,
"camera_evidence_current": camera_current,
}
def _frames() -> list[dict[str, object]]:
rows: list[dict[str, object]] = []
for frame_index in range(30):
rows.append(
{
"frame_index": frame_index,
"session_seconds": 35.0 + frame_index * 0.1,
"objects": [],
}
)
for frame_index in (0, 1, 2, 6, 7, 8):
rows[frame_index]["objects"] = [_object(10)]
rows[3]["objects"] = []
rows[4]["objects"] = []
rows[5]["objects"] = []
rows[10]["objects"] = [_object(20)]
rows[11]["objects"] = [_object(21)]
rows[12]["objects"] = [_object(30, box=[100.0, 100.0, 180.0, 220.0])]
rows[13]["objects"] = [_object(30, box=[300.0, 100.0, 380.0, 220.0])]
for offset, state in enumerate(("static", "unknown", "static", "unknown", "static")):
rows[15 + offset]["objects"] = [
_object(40, motion_state=state, world_track_id=240001 + offset % 2)
]
for frame_index in range(20, 25):
rows[frame_index]["objects"] = [_object(50, camera_current=frame_index in {20, 24})]
for offset, track_id in enumerate((60, 61, 62, 63)):
rows[26 + offset]["objects"] = [_object(track_id)]
return rows
def test_e46d_scans_observable_temporal_failures_and_ranks_video_clips() -> None:
signals, clips, metrics = analyze_temporal_frames(_frames())
kinds = {signal["kind"] for signal in signals}
assert "layer-blackout" in kinds
assert "route-layer-gap" in kinds
assert "route-id-rebirth-candidate" in kinds
assert "bbox-jump" in kinds
assert "motion-state-flap" in kinds
assert "world-binding-flap" in kinds
assert "short-track-burst" in kinds
assert metrics["temporal_continuity_passed"] is False
assert metrics["failure_signal_count"] == len(signals)
assert metrics["review_clip_count"] == len(clips)
assert clips[0]["priority"] == "critical"
assert all(clip["start_seconds"] <= clip["event_start_seconds"] for clip in clips)
assert all(clip["event_end_seconds"] <= clip["end_seconds"] for clip in clips)
def test_e46d_distinguishes_camera_hold_from_route_layer_loss() -> None:
frames = _frames()
signals, _, metrics = analyze_temporal_frames(frames)
held = [signal for signal in signals if signal["kind"] == "camera-evidence-hold"]
gaps = [signal for signal in signals if signal["kind"] == "route-layer-gap"]
assert held
assert gaps
assert all(signal["evidence"]["camera_evidence_current"] is False for signal in held)
assert all(signal["evidence"]["same_route_identity_returned"] is True for signal in gaps)
assert metrics["detector_hold_episode_count"] == len(held)
+279
View File
@@ -0,0 +1,279 @@
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import pytest
from k1link.compute.e46e_ready_stack import (
E46E_RUNTIME_SCHEMA,
E46EReadyStackError,
analyze_e46e_frames,
build_e46e_ready_stack,
read_e46e_ready_stack,
)
def test_e46e_projects_stock_deepstream_output_without_custom_tracking(
tmp_path: Path,
) -> None:
source, profile = _source_and_profile(tmp_path, frame_count=6)
raw = _raw_output(tmp_path, profile, frame_count=6)
_detector(raw, 0, "car", (10, 20, 40, 60), 0.91)
_tracker(raw, 0, 7, "car", (10, 20, 40, 60), 0.88)
_tracker(raw, 1, 7, "car", (12, 20, 42, 60), 0.73)
_detector(raw, 5, "person", (100, 80, 130, 160), 0.93)
_tracker(raw, 5, 7, "person", (100, 80, 130, 160), 0.79)
result = build_e46e_ready_stack(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
metrics = result["report"]["metrics"]
assert metrics["frame_count"] == 6
assert metrics["detection_observation_count"] == 2
assert metrics["track_observation_count"] == 3
assert metrics["detection_box_clipped_count"] == 0
assert metrics["track_box_clipped_count"] == 0
assert metrics["tracker_recovered_frame_count"] == 1
assert metrics["full_layer_blackout_event_count"] == 1
assert metrics["route_id_gap_event_count"] == 1
assert metrics["track_class_switch_count"] == 1
assert result["frames"][1]["objects"][0]["object_id"] == "nvdcf-7"
assert result["frames"][1]["objects"][0]["bbox"] == [12.0, 20.0, 30.0, 40.0]
assert [
(component["kind"], component["role"])
for component in result["report"]["method"]["components"]
] == [
("source", "exact recorded camera evidence"),
("model", "framewise traffic-object detection"),
("tool", "official RT-DETR output decoding"),
("algorithm", "route-local temporal association"),
("runtime", "GPU inference and media pipeline"),
]
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
assert result["manifest"]["authority"]["navigation_or_safety_accepted"] is False
repeated = build_e46e_ready_stack(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
assert repeated["result_id"] == result["result_id"]
def test_e46e_clips_only_display_geometry_at_the_source_plane(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path, frame_count=1)
raw = _raw_output(tmp_path, profile, frame_count=1)
_tracker(raw, 0, 31, "person", (447, 547, 527, 608), 0.69)
result = build_e46e_ready_stack(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
item = result["frames"][0]["objects"][0]
assert item["bbox"] == [447.0, 547.0, 80.0, 53.0]
assert item["source_bbox_ltrb"] == [447.0, 547.0, 527.0, 608.0]
assert item["source_plane_clipped"] is True
assert result["report"]["metrics"]["track_box_clipped_count"] == 1
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
def test_e46e_rejects_tampered_immutable_artifact(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path, frame_count=1)
raw = _raw_output(tmp_path, profile, frame_count=1)
result = build_e46e_ready_stack(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
(result["result_root"] / "overlay.mp4").write_bytes(b"changed")
with pytest.raises(E46EReadyStackError, match="artifact changed"):
read_e46e_ready_stack(result["result_root"])
def test_e46e_frame_analyzer_requires_contiguous_source_order() -> None:
with pytest.raises(E46EReadyStackError, match="not contiguous"):
analyze_e46e_frames(
[
{
"frame_index": 1,
"session_seconds": 1.0,
"detections": [],
"objects": [],
}
]
)
def _source_and_profile(tmp_path: Path, *, frame_count: int) -> tuple[Path, Path]:
source = tmp_path / "source-job"
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
camera.mkdir(parents=True)
index_rows = [
{
"schema_version": "missioncore.camera-recording-index/v1",
"kind": "media",
"sequence": sequence,
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
}
for sequence in range(1, frame_count + 1)
]
index_path = camera / "index.jsonl"
index_path.write_text(
"".join(json.dumps(row, sort_keys=True) + "\n" for row in index_rows),
encoding="utf-8",
)
stream_sha = "1" * 64
summary_path = camera / "summary.json"
summary_path.write_text(
json.dumps(
{
"schema_version": "missioncore.camera-recording/v1",
"stream_sha256": stream_sha,
"segment_count": frame_count,
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
index_sha = _sha(index_path)
summary_sha = _sha(summary_path)
job = {
"schema_version": "missioncore.compute-job/v1",
"job_id": "recorded-test",
"input": {
"session_id": "test-session",
"source_id": "sensor.camera.right",
"segment_count": frame_count,
"archive_index_sha256": index_sha,
"archive_summary_sha256": summary_sha,
"timeline": {"start_seconds": 10.0, "end_seconds": 11.0},
},
}
(source / "job.json").write_text(json.dumps(job, indent=2) + "\n", encoding="utf-8")
profile_value = {
"schema_version": "missioncore.e46e-ready-stack-profile/v1",
"profile_id": "e46e-test/v1",
"source": {
"camera_source_id": "sensor.camera.right",
"job_id": "recorded-test",
"session_id": "test-session",
"segment_count": frame_count,
"stream_sha256": stream_sha,
"archive_index_sha256": index_sha,
"archive_summary_sha256": summary_sha,
},
"runtime": {
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:"
+ "2" * 64,
"deepstream_version": "9.1",
},
"detector": {
"name": "NVIDIA TrafficCamNet Transformer Lite",
"version": "test",
"model_sha256": "3" * 64,
"custom_postprocessing": False,
},
"parser": {
"name": "NVIDIA DeepStream TAO custom bounding-box parser",
"repository": "https://github.com/NVIDIA/DeepStream.git",
"commit": "8" * 40,
"symbol": "NvDsInferParseCustomDDETRTAO",
"library_sha256": "9" * 64,
"custom_mission_core_logic": False,
},
"tracker": {
"name": "NVIDIA NvDCF",
"configuration": "stock-accuracy",
"custom_association": False,
"custom_hold_or_stitch": False,
},
"output": {"frame_width": 800, "frame_height": 600},
"authority": {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
return source, profile
def _raw_output(tmp_path: Path, profile_path: Path, *, frame_count: int) -> Path:
raw = tmp_path / "raw"
detections = raw / "detections"
tracks = raw / "tracks"
detections.mkdir(parents=True)
tracks.mkdir()
for frame in range(frame_count):
(detections / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
(tracks / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
overlay = raw / "overlay.mp4"
overlay.write_bytes(b"synthetic-overlay")
(raw / "deepstream.log").write_text("synthetic success\n", encoding="utf-8")
profile = json.loads(profile_path.read_text(encoding="utf-8"))
image = profile["runtime"]["container_image"]
runtime = {
"schema_version": E46E_RUNTIME_SCHEMA,
"status": "completed",
"worker_host": "TEST-WORKER-006",
"gpu_name": "Synthetic RTX",
"container_image": image,
"container_image_digest": image.rsplit("@sha256:", 1)[1],
"model_sha256": profile["detector"]["model_sha256"],
"model_engine_sha256": "4" * 64,
"deepstream_config_sha256": "5" * 64,
"detector_config_sha256": "6" * 64,
"parser_library_sha256": profile["parser"]["library_sha256"],
"tracker_config_sha256": "7" * 64,
"input_stream_sha256": profile["source"]["stream_sha256"],
"overlay_sha256": _sha(overlay),
}
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
return raw
def _detector(
raw: Path,
frame: int,
label: str,
box: tuple[int, int, int, int],
confidence: float,
) -> None:
left, top, right, bottom = box
(raw / "detections" / f"00_000_{frame:06d}.txt").write_text(
f"{label} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
encoding="utf-8",
)
def _tracker(
raw: Path,
frame: int,
track_id: int,
label: str,
box: tuple[int, int, int, int],
confidence: float,
) -> None:
left, top, right, bottom = box
(raw / "tracks" / f"00_000_{frame:06d}.txt").write_text(
f"{label} {track_id} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
encoding="utf-8",
)
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import sys
from pathlib import Path
import pytest
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e46e_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e46e_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e46e_package_is_minimal_deterministic_and_content_addressed(
tmp_path: Path,
) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
profile, parser_library = _package_inputs(tmp_path, repository)
kwargs = {
"repository_root": repository,
"profile_path": profile,
"parser_library_path": parser_library,
"output_root": tmp_path,
}
package = module.build_e46e_worker_package(**kwargs)
repeated = module.build_e46e_worker_package(**kwargs)
manifest = module.validate_e46e_worker_package(package)
assert repeated == package
assert package.name == f"e46e-worker-package-{manifest['identity_sha256']}"
assert manifest["identity"]["classification"] == (
"minimal-stock-nvidia-recorded-right-worker-package"
)
assert manifest["identity"]["tracker"]["custom_association"] is False
assert manifest["identity"]["tracker"]["custom_hold_or_stitch"] is False
assert manifest["identity"]["parser"]["custom_mission_core_logic"] is False
assert len(manifest["artifacts"]) == 13
def test_e46e_package_rejects_unexpected_member(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
profile, parser_library = _package_inputs(tmp_path, repository)
package = module.build_e46e_worker_package(
repository_root=repository,
profile_path=profile,
parser_library_path=parser_library,
output_root=tmp_path,
)
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
with pytest.raises(module.E46EWorkerPackageError, match="file set changed"):
module.validate_e46e_worker_package(package)
def _package_inputs(tmp_path: Path, repository: Path) -> tuple[Path, Path]:
parser_library = tmp_path / "libnvds_infercustomparser_tao.so"
parser_library.write_bytes(b"synthetic official parser fixture")
profile_value = json.loads(
(
repository
/ "experiments"
/ "perception"
/ "e46e_ready_stack_profile.json"
).read_text(encoding="utf-8")
)
profile_value["parser"]["library_sha256"] = hashlib.sha256(
parser_library.read_bytes()
).hexdigest()
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
return profile, parser_library
+227
View File
@@ -0,0 +1,227 @@
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import pytest
from k1link.compute.e46f_dashcam_bakeoff import (
E46F_RUNTIME_SCHEMA,
E46FDashCamBakeoffError,
build_e46f_dashcam_bakeoff,
read_e46f_dashcam_bakeoff,
)
def test_e46f_freezes_only_the_stock_detector_change(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path, frame_count=3)
raw = _raw_output(tmp_path, profile, frame_count=3)
_observation(raw, "detections", 0, "car", (10, 20, 40, 60), 0.91)
_observation(raw, "tracks", 0, "car", (10, 20, 40, 60), 0.88, track_id=7)
_observation(raw, "tracks", 1, "car", (12, 20, 42, 60), 0.73, track_id=7)
_observation(raw, "detections", 2, "person", (100, 80, 130, 160), 0.93)
_observation(raw, "tracks", 2, "person", (100, 80, 130, 160), 0.79, track_id=8)
result = build_e46f_dashcam_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
metrics = result["report"]["metrics"]
assert metrics["frame_count"] == 3
assert metrics["detection_observation_count"] == 2
assert metrics["track_observation_count"] == 3
assert metrics["tracker_recovered_frame_count"] == 1
assert result["frames"][0]["detections"][0]["provenance"] == ("nvidia-dashcamnet-detectnet-v2")
assert result["report"]["comparison_contract"] == {
"baseline_result_id": f"e46e-ready-stack-{'a' * 64}",
"controlled_change": "detector-only",
"held_constant": ["recorded RIGHT source", "DeepStream", "FP16", "NvDCF"],
}
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
assert result["manifest"]["authority"]["navigation_or_safety_accepted"] is False
repeated = build_e46f_dashcam_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
assert repeated["result_id"] == result["result_id"]
def test_e46f_rejects_tampered_immutable_video(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path, frame_count=1)
raw = _raw_output(tmp_path, profile, frame_count=1)
result = build_e46f_dashcam_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
(result["result_root"] / "overlay.mp4").write_bytes(b"changed")
with pytest.raises(E46FDashCamBakeoffError, match="artifact changed"):
read_e46f_dashcam_bakeoff(result["result_root"])
def _source_and_profile(tmp_path: Path, *, frame_count: int) -> tuple[Path, Path]:
source = tmp_path / "source-job"
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
camera.mkdir(parents=True)
rows = [
{
"schema_version": "missioncore.camera-recording-index/v1",
"kind": "media",
"sequence": sequence,
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
}
for sequence in range(1, frame_count + 1)
]
index_path = camera / "index.jsonl"
index_path.write_text(
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
encoding="utf-8",
)
stream_sha = "1" * 64
summary_path = camera / "summary.json"
summary_path.write_text(
json.dumps(
{
"schema_version": "missioncore.camera-recording/v1",
"stream_sha256": stream_sha,
"segment_count": frame_count,
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
index_sha = _sha(index_path)
summary_sha = _sha(summary_path)
(source / "job.json").write_text(
json.dumps(
{
"schema_version": "missioncore.compute-job/v1",
"job_id": "recorded-test",
"input": {
"session_id": "test-session",
"source_id": "sensor.camera.right",
"segment_count": frame_count,
"archive_index_sha256": index_sha,
"archive_summary_sha256": summary_sha,
"timeline": {"start_seconds": 10.0, "end_seconds": 11.0},
},
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
profile_value = {
"schema_version": "missioncore.e46f-dashcam-bakeoff-profile/v1",
"profile_id": "e46f-test/v1",
"comparison_contract": {
"baseline_result_id": f"e46e-ready-stack-{'a' * 64}",
"controlled_change": "detector-only",
"held_constant": ["recorded RIGHT source", "DeepStream", "FP16", "NvDCF"],
},
"source": {
"camera_source_id": "sensor.camera.right",
"job_id": "recorded-test",
"session_id": "test-session",
"segment_count": frame_count,
"stream_sha256": stream_sha,
"archive_index_sha256": index_sha,
"archive_summary_sha256": summary_sha,
},
"runtime": {
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:" + "2" * 64,
"deepstream_version": "9.1",
},
"detector": {
"name": "NVIDIA DashCamNet",
"version": "pruned_onnx_v1.0.4",
"model_sha256": "3" * 64,
"custom_postprocessing": False,
},
"postprocessor": {
"name": "NVIDIA DeepStream built-in DetectNet_v2 parser and NMS",
"cluster_mode": "NMS",
"reference_commit": "4" * 40,
"reference_config_sha256": "5" * 64,
"custom_mission_core_logic": False,
},
"tracker": {
"name": "NVIDIA NvDCF",
"configuration": "stock-performance",
"custom_association": False,
"custom_hold_or_stitch": False,
},
"output": {"frame_width": 800, "frame_height": 600},
"authority": {
"ground_truth": False,
"candidate_accepted": False,
"commands_enabled": False,
"navigation_or_safety_accepted": False,
},
}
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
return source, profile
def _raw_output(tmp_path: Path, profile_path: Path, *, frame_count: int) -> Path:
raw = tmp_path / "raw"
for name in ("detections", "tracks"):
directory = raw / name
directory.mkdir(parents=True, exist_ok=True)
for frame in range(frame_count):
(directory / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
overlay = raw / "overlay.mp4"
overlay.write_bytes(b"synthetic-overlay")
(raw / "deepstream.log").write_text("synthetic success\n", encoding="utf-8")
profile = json.loads(profile_path.read_text(encoding="utf-8"))
image = profile["runtime"]["container_image"]
runtime = {
"schema_version": E46F_RUNTIME_SCHEMA,
"status": "completed",
"worker_host": "TEST-WORKER-006",
"gpu_name": "Synthetic RTX",
"container_image": image,
"container_image_digest": image.rsplit("@sha256:", 1)[1],
"model_sha256": profile["detector"]["model_sha256"],
"model_engine_sha256": "6" * 64,
"deepstream_config_sha256": "7" * 64,
"detector_config_sha256": "8" * 64,
"tracker_config_sha256": "9" * 64,
"input_stream_sha256": profile["source"]["stream_sha256"],
"overlay_sha256": _sha(overlay),
}
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
return raw
def _observation(
raw: Path,
directory: str,
frame: int,
label: str,
box: tuple[int, int, int, int],
confidence: float,
*,
track_id: int | None = None,
) -> None:
left, top, right, bottom = box
identity = "" if track_id is None else f" {track_id}"
(raw / directory / f"00_000_{frame:06d}.txt").write_text(
f"{label}{identity} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
encoding="utf-8",
)
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e46f_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e46f_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e46f_package_is_minimal_deterministic_and_detector_only(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
kwargs = {
"repository_root": repository,
"profile_path": repository
/ "experiments"
/ "perception"
/ "e46f_dashcam_bakeoff_profile.json",
"output_root": tmp_path,
}
package = module.build_e46f_worker_package(**kwargs)
repeated = module.build_e46f_worker_package(**kwargs)
manifest = module.validate_e46f_worker_package(package)
assert repeated == package
assert package.name == f"e46f-worker-package-{manifest['identity_sha256']}"
assert manifest["identity"]["classification"] == (
"minimal-stock-nvidia-detector-only-bakeoff-package"
)
assert manifest["identity"]["baseline_result_id"].startswith("e46e-ready-stack-")
assert manifest["identity"]["tracker"]["custom_association"] is False
assert manifest["identity"]["tracker"]["custom_hold_or_stitch"] is False
assert manifest["identity"]["postprocessor"]["custom_mission_core_logic"] is False
assert len(manifest["artifacts"]) == 11
def test_e46f_package_rejects_unexpected_member(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
package = module.build_e46f_worker_package(
repository_root=repository,
profile_path=repository
/ "experiments"
/ "perception"
/ "e46f_dashcam_bakeoff_profile.json",
output_root=tmp_path,
)
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
with pytest.raises(module.E46FWorkerPackageError, match="file set changed"):
module.validate_e46f_worker_package(package)
@@ -0,0 +1,258 @@
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import pytest
from k1link.compute.e46g_rectified_detector_bakeoff import (
E46G_RUNTIME_SCHEMA,
E46GRectifiedDetectorBakeoffError,
build_e46g_rectified_detector_bakeoff,
read_e46g_rectified_detector_bakeoff,
)
def test_e46g_freezes_same_calibrated_views_for_both_stock_detectors(
tmp_path: Path,
) -> None:
source, profile = _source_and_profile(tmp_path)
raw = _raw_output(tmp_path, profile)
result = build_e46g_rectified_detector_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
report = result["report"]
assert report["status"] == "completed-awaiting-visual-semantic-adjudication"
assert report["acceptance"]["official_nvidia_dewarper_executed"] is True
assert report["acceptance"]["same_views_and_frames_for_both_candidates"] is True
assert report["decision"]["automatic_winner_selected"] is False
assert report["decision"]["custom_detector_or_tracker_logic_used"] is False
assert {component["kind"] for component in report["method"]["components"]} <= {
"source",
"tool",
"model",
"algorithm",
"runtime",
}
assert set(report["metrics"]) == {"trafficcamnet", "dashcamnet"}
assert report["metrics"]["trafficcamnet"]["view_frame_count"] == 6
assert report["metrics"]["trafficcamnet"]["track_observation_count"] == 3
assert report["metrics"]["dashcamnet"]["track_observation_count"] == 3
assert result["frames"]["trafficcamnet-front"][0]["source_frame_index"] == 0
assert (
result["frames"]["dashcamnet-right"][0]["objects"][0]["object_id"]
== "dashcamnet-right-nvdcf-7"
)
assert result["manifest"]["authority"]["candidate_accepted"] is False
repeated = build_e46g_rectified_detector_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
assert repeated["result_id"] == result["result_id"]
def test_e46g_rejects_tampered_comparison_video(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path)
raw = _raw_output(tmp_path, profile)
result = build_e46g_rectified_detector_bakeoff(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
result["comparison_paths"]["trafficcamnet"].write_bytes(b"changed")
with pytest.raises(E46GRectifiedDetectorBakeoffError, match="artifact changed"):
read_e46g_rectified_detector_bakeoff(result["result_root"])
def _source_and_profile(tmp_path: Path) -> tuple[Path, Path]:
repository_profile = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "e46g_rectified_detector_bakeoff_profile.json"
)
profile_value = json.loads(repository_profile.read_text(encoding="utf-8"))
source = tmp_path / "source-job"
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
camera.mkdir(parents=True)
rows = [
{
"schema_version": "missioncore.camera-recording-index/v1",
"kind": "media",
"sequence": sequence,
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
}
for sequence in range(1, 4490)
]
index_path = camera / "index.jsonl"
index_path.write_text(
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
encoding="utf-8",
)
summary_path = camera / "summary.json"
summary_path.write_text(
json.dumps(
{
"schema_version": "missioncore.camera-recording/v1",
"stream_sha256": "1" * 64,
"segment_count": 4489,
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
profile_value["source"].update(
{
"job_id": "recorded-test",
"session_id": "test-session",
"stream_sha256": "1" * 64,
"archive_index_sha256": _sha(index_path),
"archive_summary_sha256": _sha(summary_path),
}
)
profile_value["selection"].update(
{
"first_source_frame_index": 0,
"last_source_frame_index": 1,
"frame_count": 2,
}
)
(source / "job.json").write_text(
json.dumps(
{
"schema_version": "missioncore.compute-job/v1",
"job_id": "recorded-test",
"input": {
"session_id": "test-session",
"source_id": "sensor.camera.right",
"segment_count": 4489,
"archive_index_sha256": _sha(index_path),
"archive_summary_sha256": _sha(summary_path),
"timeline": {"start_seconds": 10.0, "end_seconds": 459.0},
},
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
return source, profile
def _raw_output(tmp_path: Path, profile_path: Path) -> Path:
profile = json.loads(profile_path.read_text(encoding="utf-8"))
raw = tmp_path / "raw"
geometry_runtime: dict[str, object] = {}
for view in ("left", "front", "right"):
full = _file(raw / "geometry" / f"{view}.mp4", f"full-{view}")
sample = _file(raw / "samples" / f"{view}.mp4", f"sample-{view}")
geometry_runtime[view] = {
"dewarper_config_sha256": profile["rectification"]["views"][view]["config_sha256"],
"dewarper_log_sha256": "2" * 64,
"full_rectified_video_path": f"geometry/{view}.mp4",
"full_rectified_video_sha256": _sha(full),
"sample_video_path": f"samples/{view}.mp4",
"sample_video_sha256": _sha(sample),
"full_frame_count": profile["rectification"]["expected_full_frame_count"],
"retained_source_frame_index_range": profile["rectification"][
"retained_source_frame_index_range"
],
"excluded_source_tail_frame_count": profile["rectification"][
"excluded_source_tail_frame_count"
],
"sample_frame_count": 2,
}
candidates_runtime: dict[str, object] = {}
for candidate in ("trafficcamnet", "dashcamnet"):
runs: dict[str, object] = {}
for view in ("left", "front", "right"):
root = raw / "runs" / candidate / view
for directory in ("detections", "tracks"):
(root / directory).mkdir(parents=True, exist_ok=True)
for frame in range(2):
target = root / directory / f"00_000_{frame:06d}.txt"
target.write_text(
_kitti_row(directory == "tracks") if frame == 0 else "",
encoding="utf-8",
)
overlay = _file(root / "overlay.mp4", f"overlay-{candidate}-{view}")
log = _file(root / "deepstream.log", "success")
runs[view] = {
"overlay_path": f"runs/{candidate}/{view}/overlay.mp4",
"overlay_sha256": _sha(overlay),
"deepstream_log_path": f"runs/{candidate}/{view}/deepstream.log",
"deepstream_log_sha256": _sha(log),
"tracker_config_sha256": "3" * 64,
"model_engine_sha256": "4" * 64,
"frame_count": 2,
"deepstream_exit_code": 0,
}
candidates_runtime[candidate] = {
"model_sha256": profile["candidates"][candidate]["model_sha256"],
"deepstream_app_config_sha256": profile["candidates"][candidate][
"deepstream_app_config_sha256"
],
"detector_config_sha256": profile["candidates"][candidate]["detector_config_sha256"],
"parser_library_sha256": (
profile["trafficcamnet_parser"]["library_sha256"]
if candidate == "trafficcamnet"
else None
),
"runs": runs,
}
comparison_runtime: dict[str, object] = {}
for candidate in ("trafficcamnet", "dashcamnet"):
video = _file(raw / "comparison" / f"{candidate}.mp4", candidate)
comparison_runtime[candidate] = {
"video_path": f"comparison/{candidate}.mp4",
"video_sha256": _sha(video),
"frame_count": 2,
"view_order": ["left", "front", "right"],
}
_file(raw / "worker.log", "completed")
image = profile["runtime"]["container_image"]
runtime = {
"schema_version": E46G_RUNTIME_SCHEMA,
"status": "completed",
"worker_host": "TEST-WORKER-006",
"gpu_name": "Synthetic RTX",
"container_image": image,
"container_image_digest": image.rsplit("@sha256:", 1)[1],
"source_stream_sha256": profile["source"]["stream_sha256"],
"first_source_frame_index": 0,
"sample_frame_count": 2,
"geometry": geometry_runtime,
"candidates": candidates_runtime,
"comparison": comparison_runtime,
}
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
return raw
def _file(path: Path, value: str) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(value.encode())
return path
def _kitti_row(tracked: bool) -> str:
identity = " 7" if tracked else ""
return f"car{identity} 0.0 0 0.0 10 20 300 400 0 0 0 0 0 0 0 0.91\n"
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
+87
View File
@@ -0,0 +1,87 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
import pytest
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e46g_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e46g_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e46g_package_is_deterministic_calibrated_and_stock(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
kwargs = {
"repository_root": repository,
"profile_path": repository
/ "experiments"
/ "perception"
/ "e46g_rectified_detector_bakeoff_profile.json",
"parser_library_path": repository
/ ".runtime"
/ "compute-experiments"
/ "e46e"
/ "parser"
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
/ "libnvds_infercustomparser_tao.so",
"output_root": tmp_path,
}
package = module.build_e46g_worker_package(**kwargs)
repeated = module.build_e46g_worker_package(**kwargs)
manifest = module.validate_e46g_worker_package(package)
assert repeated == package
assert package.name == f"e46g-worker-package-{manifest['identity_sha256']}"
identity = manifest["identity"]
assert identity["classification"] == (
"minimal-factory-kb4-stock-nvidia-detector-bakeoff-package"
)
assert identity["calibration"]["calibration_sha256"] == (
"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
)
assert identity["rectification"]["provider"] == "NVIDIA Gst-nvdewarper"
assert identity["rectification"]["expected_full_frame_count"] == 4488
assert identity["rectification"]["retained_source_frame_index_range"] == [0, 4487]
assert identity["selection"]["frame_count"] == 600
assert all(
candidate["custom_postprocessing"] is False for candidate in identity["candidates"].values()
)
assert identity["tracker"]["custom_association"] is False
assert identity["tracker"]["custom_hold_or_stitch"] is False
assert len(manifest["artifacts"]) == 19
def test_e46g_package_rejects_unexpected_member(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
package = module.build_e46g_worker_package(
repository_root=repository,
profile_path=repository
/ "experiments"
/ "perception"
/ "e46g_rectified_detector_bakeoff_profile.json",
parser_library_path=repository
/ ".runtime"
/ "compute-experiments"
/ "e46e"
/ "parser"
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
/ "libnvds_infercustomparser_tao.so",
output_root=tmp_path,
)
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
with pytest.raises(module.E46GWorkerPackageError, match="file set changed"):
module.validate_e46g_worker_package(package)
@@ -0,0 +1,167 @@
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import pytest
from k1link.compute.e46h_full_rectified_front_replay import (
E46H_RUNTIME_SCHEMA,
E46HFullRectifiedFrontReplayError,
build_e46h_full_rectified_front_replay,
read_e46h_full_rectified_front_replay,
)
def test_e46h_freezes_full_retained_front_route(tmp_path: Path) -> None:
source, profile = _source_and_profile(tmp_path)
raw = _raw_output(tmp_path, profile)
result = build_e46h_full_rectified_front_replay(
source_job_root=source,
raw_root=raw,
profile_path=profile,
output_root=tmp_path / "results",
)
report = result["report"]
assert report["status"] == "completed-awaiting-full-route-visual-review"
assert report["metrics"]["frame_count"] == 4488
assert report["acceptance"]["retained_route_accounted"] is True
assert report["acceptance"]["terminal_source_frame_excluded"] is True
assert report["acceptance"]["full_visual_review_completed"] is False
assert report["decision"]["selected_provider"] == "front-trafficcamnet-stock-nvdcf"
assert report["decision"]["custom_detector_or_tracker_logic_used"] is False
assert result["frames"][0]["source_frame_index"] == 0
assert result["frames"][-1]["source_frame_index"] == 4487
assert result["manifest"]["authority"]["candidate_accepted"] is False
result["overlay_path"].write_bytes(b"changed")
with pytest.raises(E46HFullRectifiedFrontReplayError, match="artifact changed"):
read_e46h_full_rectified_front_replay(result["result_root"])
def _source_and_profile(tmp_path: Path) -> tuple[Path, Path]:
repository = Path(__file__).resolve().parents[1]
profile_value = json.loads(
(
repository
/ "experiments"
/ "perception"
/ "e46h_full_rectified_front_replay_profile.json"
).read_text(encoding="utf-8")
)
source = tmp_path / "source-job"
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
camera.mkdir(parents=True)
rows = [
{
"schema_version": "missioncore.camera-recording-index/v1",
"kind": "media",
"sequence": sequence,
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
}
for sequence in range(1, 4490)
]
index_path = camera / "index.jsonl"
index_path.write_text(
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
encoding="utf-8",
)
summary_path = camera / "summary.json"
summary_path.write_text(
json.dumps({"stream_sha256": "1" * 64, "segment_count": 4489}) + "\n",
encoding="utf-8",
)
profile_value["source"].update(
{
"job_id": "recorded-test",
"session_id": "test-session",
"stream_sha256": "1" * 64,
"archive_index_sha256": _sha(index_path),
"archive_summary_sha256": _sha(summary_path),
}
)
(source / "job.json").write_text(
json.dumps(
{
"schema_version": "missioncore.compute-job/v1",
"job_id": "recorded-test",
"input": {
"session_id": "test-session",
"source_id": "sensor.camera.right",
"segment_count": 4489,
"archive_index_sha256": _sha(index_path),
"archive_summary_sha256": _sha(summary_path),
"timeline": {"start_seconds": 10.0, "end_seconds": 459.0},
},
}
)
+ "\n",
encoding="utf-8",
)
profile = tmp_path / "profile.json"
profile.write_text(json.dumps(profile_value) + "\n", encoding="utf-8")
return source, profile
def _raw_output(tmp_path: Path, profile_path: Path) -> Path:
profile = json.loads(profile_path.read_text(encoding="utf-8"))
raw = tmp_path / "raw"
front = _file(raw / "geometry" / "front.mp4", "front")
front_log = _file(raw / "geometry" / "front.log", "dewarper")
overlay = _file(raw / "run" / "overlay.mp4", "overlay")
deepstream_log = _file(raw / "run" / "deepstream.log", "deepstream")
_file(raw / "worker.log", "completed")
for directory in (raw / "run" / "detections", raw / "run" / "tracks"):
directory.mkdir(parents=True)
for frame in range(4488):
(directory / f"00_000_{frame:06d}.txt").touch()
image = profile["runtime"]["container_image"]
runtime = {
"schema_version": E46H_RUNTIME_SCHEMA,
"status": "completed",
"worker_host": "TEST-WORKER-006",
"gpu_name": "Synthetic RTX",
"container_image": image,
"container_image_digest": image.rsplit("@sha256:", 1)[1],
"source_stream_sha256": profile["source"]["stream_sha256"],
"frame_count": 4488,
"retained_source_frame_index_range": [0, 4487],
"geometry": {
"video_path": "geometry/front.mp4",
"video_sha256": _sha(front),
"log_path": "geometry/front.log",
"log_sha256": _sha(front_log),
"config_sha256": profile["rectification"]["config_sha256"],
"frame_count": 4488,
},
"run": {
"overlay_path": "run/overlay.mp4",
"overlay_sha256": _sha(overlay),
"deepstream_log_path": "run/deepstream.log",
"deepstream_log_sha256": _sha(deepstream_log),
"model_sha256": profile["detector"]["model_sha256"],
"model_engine_sha256": "2" * 64,
"parser_library_sha256": profile["parser"]["library_sha256"],
"deepstream_app_config_sha256": profile["detector"][
"deepstream_app_config_sha256"
],
"detector_config_sha256": profile["detector"]["detector_config_sha256"],
"tracker_config_sha256": "3" * 64,
"frame_count": 4488,
},
}
(raw / "runtime.json").write_text(json.dumps(runtime) + "\n", encoding="utf-8")
return raw
def _file(path: Path, value: str) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(value, encoding="utf-8")
return path
def _sha(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
def _module() -> object:
path = (
Path(__file__).resolve().parents[1]
/ "experiments"
/ "perception"
/ "prepare_e46h_worker_package.py"
)
spec = importlib.util.spec_from_file_location("e46h_worker_package_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_e46h_package_is_deterministic_front_only_and_stock(tmp_path: Path) -> None:
module = _module()
repository = Path(__file__).resolve().parents[1]
kwargs = {
"repository_root": repository,
"profile_path": repository
/ "experiments"
/ "perception"
/ "e46h_full_rectified_front_replay_profile.json",
"parser_library_path": repository
/ ".runtime"
/ "compute-experiments"
/ "e46e"
/ "parser"
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
/ "libnvds_infercustomparser_tao.so",
"output_root": tmp_path,
}
package = module.build_e46h_worker_package(**kwargs)
repeated = module.build_e46h_worker_package(**kwargs)
manifest = module.validate_e46h_worker_package(package)
assert repeated == package
assert package.name == f"e46h-worker-package-{manifest['identity_sha256']}"
identity = manifest["identity"]
assert identity["classification"] == "minimal-full-front-stock-nvidia-replay-package"
assert identity["rectification"]["view"] == "front"
assert identity["selection"]["frame_count"] == 4488
assert identity["selection"]["last_source_frame_index"] == 4487
assert identity["detector"]["custom_postprocessing"] is False
assert identity["tracker"]["custom_association"] is False
assert identity["tracker"]["custom_hold_or_stitch"] is False
@@ -0,0 +1,90 @@
from __future__ import annotations
import pytest
from k1link.compute.l34_right_yolox_truth_island_freeze import (
L34_PREDICTION_SCHEMA,
L34RightYoloxTruthIslandError,
freeze_l34_candidate_predictions,
)
def _reference(frame_index: int, sequence: int) -> dict[str, object]:
return {
"truth_island_sequence": sequence,
"image_id": sequence + 10,
"frame_index": frame_index,
"group_id": "temporal-a",
"session_seconds": 42.5,
"sha256": "a" * 64,
}
def _frame(frame_index: int) -> dict[str, object]:
return {
"schema_version": "missioncore.rectified-yolox-frame/v1",
"frame_index": frame_index,
"detections": [
{
"label": "bus",
"score": 0.91,
"bbox_xyxy": [10.0, 20.0, 30.0, 40.0],
},
{
"label": "person",
"score": 0.72,
"bbox_xyxy": [1.0, 2.0, 3.0, 4.0],
},
{
"label": "car",
"score": 0.24,
"bbox_xyxy": [5.0, 6.0, 7.0, 8.0],
},
],
}
def test_freezes_only_admitted_right_camera_candidate_classes() -> None:
predictions = freeze_l34_candidate_predictions(
references=(_reference(70, 1),),
detector_frames={70: _frame(70)},
minimum_score=0.25,
)
assert predictions == (
{
"schema_version": L34_PREDICTION_SCHEMA,
"candidate_id": "yolox-s-kb4-core3",
"truth_island_sequence": 1,
"image_id": 11,
"frame_index": 70,
"session_seconds": 42.5,
"source_image_sha256": "a" * 64,
"group_id": "temporal-a",
"predictions": [
{
"label": "heavy_vehicle",
"score": 0.91,
"bbox_xyxy": [10.0, 20.0, 30.0, 40.0],
},
{
"label": "person",
"score": 0.72,
"bbox_xyxy": [1.0, 2.0, 3.0, 4.0],
},
],
"truth_joined": False,
},
)
def test_freeze_fails_closed_on_incomplete_frame_coverage() -> None:
with pytest.raises(
L34RightYoloxTruthIslandError,
match="coverage is incomplete",
):
freeze_l34_candidate_predictions(
references=(_reference(70, 1), _reference(71, 2)),
detector_frames={70: _frame(70)},
minimum_score=0.25,
)
@@ -0,0 +1,115 @@
from __future__ import annotations
from k1link.compute.l34a_assisted_yolox_error_audit import (
_aggregate,
_audit_case,
)
def _prediction_row() -> dict[str, object]:
return {
"truth_island_sequence": 1,
"image_id": 2,
"frame_index": 70,
"group_id": "anchor-a",
"session_seconds": 4.2,
"source_image_sha256": "a" * 64,
"predictions": [
{
"label": "car",
"score": 0.9,
"bbox_xyxy": [100.0, 100.0, 300.0, 300.0],
},
{
"label": "car",
"score": 0.7,
"bbox_xyxy": [120.0, 120.0, 280.0, 280.0],
},
{
"label": "motorcycle",
"score": 0.6,
"bbox_xyxy": [400.0, 200.0, 520.0, 420.0],
},
],
}
def _annotation_frame() -> dict[str, object]:
return {
"truth_island_sequence": 1,
"image_id": 2,
"frame_index": 70,
"source_sha256": "a" * 64,
"objects": [
{
"object_id": "car-1",
"category": "car",
"proposed_label": None,
"origin": "frozen_candidate_seed",
"box_xyxy": [100.0, 100.0, 300.0, 300.0],
"occluded": False,
"truncated": False,
},
{
"object_id": "stroller-1",
"category": "unmapped",
"proposed_label": "Детская коляска",
"origin": "manual",
"box_xyxy": [400.0, 200.0, 520.0, 420.0],
"occluded": False,
"truncated": False,
},
{
"object_id": "person-1",
"category": "person",
"proposed_label": None,
"origin": "manual",
"box_xyxy": [10.0, 10.0, 60.0, 160.0],
"occluded": False,
"truncated": False,
},
],
}
def test_assisted_audit_distinguishes_duplicate_mismatch_and_miss() -> None:
case = _audit_case(
prediction_row=_prediction_row(),
annotation_frame=_annotation_frame(),
)
assert case["summary"] == {
"prediction_count": 3,
"reference_count": 3,
"true_positive": 1,
"false_positive": 2,
"false_negative": 2,
"class_mismatch": 1,
"duplicate_false_positive": 1,
"unmatched_false_positive": 0,
"unmatched_false_negative": 1,
"severity_score": 7,
}
assert [item["verdict"] for item in case["predictions"]] == [
"true_positive",
"duplicate_false_positive",
"class_mismatch",
]
assert case["annotations"][1]["display_category"] == (
"unmapped:Детская коляска"
)
def test_assisted_alignment_metrics_remain_descriptive() -> None:
case = _audit_case(
prediction_row=_prediction_row(),
annotation_frame=_annotation_frame(),
)
metrics = _aggregate((case,))
assert metrics["precision_iou50"] == 1 / 3
assert metrics["recall_iou50"] == 1 / 3
assert metrics["f1_iou50"] == 1 / 3
assert metrics["custom_reference_count"] == 1
assert metrics["error_case_count"] == 1
@@ -0,0 +1,100 @@
from __future__ import annotations
from k1link.compute.l34a_assisted_yolox_error_audit import _audit_case
from k1link.compute.l34b_nested_box_consolidation_shadow import (
consolidate_l34b_prediction_row,
evaluate_l34b_shadow,
)
def _row(sequence: int = 1) -> dict[str, object]:
return {
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence + 10,
"group_id": "nested-box",
"session_seconds": float(sequence),
"source_image_sha256": f"{sequence % 10}" * 64,
"predictions": [
{
"label": "heavy_vehicle",
"score": 0.71,
"bbox_xyxy": [100.0, 100.0, 140.0, 190.0],
},
{
"label": "heavy_vehicle",
"score": 0.69,
"bbox_xyxy": [90.0, 99.0, 141.0, 191.0],
},
{
"label": "car",
"score": 0.8,
"bbox_xyxy": [300.0, 100.0, 360.0, 180.0],
},
],
}
def _annotation(sequence: int = 1) -> dict[str, object]:
return {
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence + 10,
"source_sha256": f"{sequence % 10}" * 64,
"objects": [
{
"object_id": f"truck-{sequence}",
"category": "heavy_vehicle",
"proposed_label": None,
"origin": "manual",
"box_xyxy": [90.0, 99.0, 141.0, 191.0],
"occluded": False,
"truncated": False,
},
{
"object_id": f"car-{sequence}",
"category": "car",
"proposed_label": None,
"origin": "manual",
"box_xyxy": [300.0, 100.0, 360.0, 180.0],
"occluded": False,
"truncated": False,
},
],
}
def test_consolidates_only_same_category_nested_boxes() -> None:
projected, consolidations = consolidate_l34b_prediction_row(_row())
assert len(projected["predictions"]) == 2
assert consolidations[0]["source_prediction_indices"] == [1, 2]
assert projected["predictions"][0] == {
"label": "heavy_vehicle",
"score": 0.71,
"bbox_xyxy": [90.0, 99.0, 141.0, 191.0],
"source_prediction_indices": [1, 2],
}
assert projected["predictions"][1]["source_prediction_indices"] == [3]
def test_shadow_reports_regression_free_false_positive_reduction() -> None:
rows = tuple(_row(sequence) for sequence in range(1, 33))
annotations = tuple(_annotation(sequence) for sequence in range(1, 33))
before = tuple(
_audit_case(prediction_row=row, annotation_frame=annotation)
for row, annotation in zip(rows, annotations, strict=True)
)
cases, metrics = evaluate_l34b_shadow(
prediction_rows=rows,
annotation_frames=annotations,
before_cases=before,
)
assert len(cases) == 32
assert metrics["consolidation_count"] == 32
assert metrics["delta"]["true_positive"] == 0
assert metrics["delta"]["false_positive"] == -32
assert metrics["delta"]["false_negative"] == 0
assert metrics["assisted_regression_free"] is True
+153
View File
@@ -0,0 +1,153 @@
from __future__ import annotations
import copy
import pytest
from k1link.compute.l34c_tile_seam_stitch_shadow import (
L34C_PROVENANCE_SCHEMA,
L34CTileSeamStitchError,
apply_l34c_stitches,
bind_l34c_prediction_provenance,
find_l34c_temporal_stitches,
)
def _frozen_row(*, sequence: int = 1, frame_index: int = 100) -> dict[str, object]:
return {
"schema_version": "missioncore.l34-right-yolox-truth-island-prediction/v1",
"candidate_id": "yolox-s-kb4-core3",
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": frame_index,
"session_seconds": frame_index / 10.0,
"source_image_sha256": "a" * 64,
"group_id": "clip-close-car",
"predictions": [
{
"label": "car",
"score": 0.95,
"bbox_xyxy": [230.0, 220.0, 385.0, 340.0],
},
{
"label": "car",
"score": 0.90,
"bbox_xyxy": [182.0, 210.0, 266.0, 338.0],
},
],
"truth_joined": False,
}
def _detector_frame(frame_index: int = 100) -> dict[str, object]:
return {
"schema_version": "missioncore.rectified-yolox-frame/v1",
"frame_index": frame_index,
"detections": [
{
"label": "car",
"class_id": 2,
"score": 0.95,
"bbox_xyxy": [230.0, 220.0, 385.0, 340.0],
"raw_center_xy": [290.0, 280.0],
"rectification_tile": "front",
"valid_fov_fraction": 1.0,
},
{
"label": "car",
"class_id": 2,
"score": 0.90,
"bbox_xyxy": [182.0, 210.0, 266.0, 338.0],
"raw_center_xy": [230.0, 275.0],
"rectification_tile": "left",
"valid_fov_fraction": 1.0,
},
],
}
def _provenance_run(length: int) -> tuple[dict[str, object], ...]:
frozen = tuple(
_frozen_row(sequence=index + 1, frame_index=100 + index)
for index in range(length)
)
detector = {
100 + index: _detector_frame(100 + index)
for index in range(length)
}
return bind_l34c_prediction_provenance(
prediction_rows=frozen,
detector_frames=detector,
)
def test_exact_join_preserves_tile_and_raw_detector_identity() -> None:
rows = _provenance_run(1)
assert rows[0]["schema_version"] == L34C_PROVENANCE_SCHEMA
assert rows[0]["provenance_join"] == "exact-label-score-bbox"
assert rows[0]["predictions"][0]["rectification_tile"] == "front"
assert rows[0]["predictions"][1]["rectification_tile"] == "left"
assert rows[0]["predictions"][0]["class_id"] == 2
assert rows[0]["predictions"][0]["raw_label"] == "car"
def test_exact_join_rejects_missing_or_ambiguous_provenance() -> None:
frame = _detector_frame()
frame["detections"].append(copy.deepcopy(frame["detections"][0]))
with pytest.raises(L34CTileSeamStitchError, match="one exact"):
bind_l34c_prediction_provenance(
prediction_rows=(_frozen_row(),),
detector_frames={100: frame},
)
def test_temporal_gate_admits_three_consecutive_frames_but_not_two() -> None:
admitted, static_count = find_l34c_temporal_stitches(_provenance_run(3))
rejected, rejected_static_count = find_l34c_temporal_stitches(
_provenance_run(2)
)
assert static_count == 3
assert set(admitted) == {1, 2, 3}
assert all(items[0]["temporal_run_length"] == 3 for items in admitted.values())
assert rejected_static_count == 2
assert rejected == {}
def test_temporal_gate_rejects_same_tile_and_small_pairs() -> None:
rows = list(_provenance_run(3))
for row in rows:
row["predictions"][1]["rectification_tile"] = "front"
same_tile, same_tile_count = find_l34c_temporal_stitches(tuple(rows))
small_rows = list(_provenance_run(3))
for row in small_rows:
row["predictions"][0]["bbox_xyxy"] = [230.0, 220.0, 260.0, 250.0]
row["predictions"][1]["bbox_xyxy"] = [220.0, 218.0, 245.0, 252.0]
small, small_count = find_l34c_temporal_stitches(tuple(small_rows))
assert same_tile_count == 0
assert same_tile == {}
assert small_count == 0
assert small == {}
def test_apply_stitch_unions_geometry_and_preserves_source_tiles() -> None:
row = _provenance_run(3)[0]
admitted, _ = find_l34c_temporal_stitches(_provenance_run(3))
projected = apply_l34c_stitches(row, admitted[1])
assert projected["predictions"] == [
{
"label": "car",
"score": 0.95,
"bbox_xyxy": [182.0, 210.0, 385.0, 340.0],
"source_prediction_indices": [1, 2],
"source_rectification_tiles": ["front", "left"],
"temporal_run_id": "clip-close-car:car:100-102",
"temporal_run_length": 3,
}
]
@@ -0,0 +1,124 @@
from __future__ import annotations
import copy
import pytest
from k1link.compute.l34c_tile_seam_stitch_shadow import (
bind_l34c_prediction_provenance,
)
from k1link.compute.l34d_cumulative_postprocessing_candidate import (
L34DCumulativeCandidateError,
compose_l34d_prediction_row,
)
def _row() -> dict[str, object]:
return {
"schema_version": "missioncore.l34-right-yolox-truth-island-prediction/v1",
"truth_island_sequence": 1,
"image_id": 1,
"frame_index": 100,
"session_seconds": 1.0,
"source_image_sha256": "a" * 64,
"group_id": "composition",
"predictions": [
{"label": "heavy_vehicle", "score": 0.71, "bbox_xyxy": [10.0, 10.0, 30.0, 40.0]},
{"label": "heavy_vehicle", "score": 0.69, "bbox_xyxy": [9.0, 9.0, 31.0, 41.0]},
{"label": "car", "score": 0.95, "bbox_xyxy": [230.0, 220.0, 385.0, 340.0]},
{"label": "car", "score": 0.90, "bbox_xyxy": [182.0, 210.0, 266.0, 338.0]},
],
}
def _detector_frame() -> dict[str, object]:
detections = []
for index, prediction in enumerate(_row()["predictions"]):
detections.append({
**prediction,
"class_id": 7 if index < 2 else 2,
"raw_center_xy": [20.0 + index, 20.0 + index],
"rectification_tile": "front" if index in (0, 2) else "left",
"valid_fov_fraction": 1.0,
})
return {
"schema_version": "missioncore.rectified-yolox-frame/v1",
"frame_index": 100,
"detections": detections,
}
def _provenance() -> dict[str, object]:
return bind_l34c_prediction_provenance(
prediction_rows=(_row(),),
detector_frames={100: _detector_frame()},
)[0]
def _operation(indices: list[int], operation_type: str) -> dict[str, object]:
predictions = _row()["predictions"]
members = [predictions[index - 1] for index in indices]
boxes = [member["bbox_xyxy"] for member in members]
operation = {
"category": members[0]["label"],
"source_prediction_indices": indices,
"source_scores": [member["score"] for member in members],
"source_boxes_xyxy": boxes,
"merged_score": max(member["score"] for member in members),
"merged_box_xyxy": [
min(box[0] for box in boxes),
min(box[1] for box in boxes),
max(box[2] for box in boxes),
max(box[3] for box in boxes),
],
}
if operation_type == "temporal-tile-seam-stitch":
operation.update({
"source_tiles": ["front", "left"],
"temporal_run_id": "composition:car:100-102",
"temporal_run_length": 3,
})
return operation
def test_composes_disjoint_nested_and_seam_operations_once() -> None:
projected, operations = compose_l34d_prediction_row(
_provenance(),
consolidations=(_operation([1, 2], "nested-box-consolidation"),),
stitches=(_operation([3, 4], "temporal-tile-seam-stitch"),),
)
assert len(projected["predictions"]) == 2
assert [
item["source_prediction_indices"] for item in projected["predictions"]
] == [[1, 2], [3, 4]]
assert [item["operation_types"] for item in projected["predictions"]] == [
["nested-box-consolidation"],
["temporal-tile-seam-stitch"],
]
assert {item["operation_type"] for item in operations} == {
"nested-box-consolidation",
"temporal-tile-seam-stitch",
}
def test_rejects_overlapping_operation_sets() -> None:
with pytest.raises(L34DCumulativeCandidateError, match="overlap"):
compose_l34d_prediction_row(
_provenance(),
consolidations=(_operation([1, 2], "nested-box-consolidation"),),
stitches=(_operation([1, 2], "temporal-tile-seam-stitch"),),
)
def test_rejects_operation_payload_drift() -> None:
operation = _operation([1, 2], "nested-box-consolidation")
operation["source_boxes_xyxy"] = copy.deepcopy(operation["source_boxes_xyxy"])
operation["source_boxes_xyxy"][0][0] += 1.0
with pytest.raises(L34DCumulativeCandidateError, match="payload"):
compose_l34d_prediction_row(
_provenance(),
consolidations=(operation,),
stitches=(),
)
+152
View File
@@ -0,0 +1,152 @@
from __future__ import annotations
from typing import Any
import pytest
from k1link.compute.l34e_self_review_diagnostic import (
L34ESelfReviewDiagnosticError,
evaluate_l34e_self_review_diagnostic,
)
def _candidate_case(
sequence: int,
*,
category: str = "car",
box: list[float] | None = None,
) -> dict[str, Any]:
predictions = [] if box is None else [
{
"prediction_index": 1,
"category": category,
"score": 0.8,
"box_xyxy": box,
"source_prediction_indices": [1],
"source_rectification_tiles": ["center"],
"operation_types": [],
}
]
return {
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence * 10,
"group_id": f"frame-{sequence:02d}",
"session_seconds": float(sequence),
"source_image_sha256": f"{sequence:064x}",
"after_predictions": predictions,
}
def _review_frame(
sequence: int,
*,
category: str = "car",
box: list[float] | None = None,
) -> dict[str, Any]:
objects = [] if box is None else [
{
"object_id": f"manual-{sequence:02d}",
"category": category,
"proposed_label": None,
"origin": "manual",
"box_xyxy": box,
"occluded": False,
"truncated": False,
}
]
return {
"truth_island_sequence": sequence,
"image_id": sequence,
"frame_index": sequence * 10,
"source_sha256": f"{sequence:064x}",
"reviewed": True,
"objects": objects,
}
def _fixture() -> tuple[tuple[dict[str, Any], ...], tuple[dict[str, Any], ...]]:
candidates = []
reviews = []
for sequence in range(1, 33):
candidate_category = "car"
review_category = "car"
candidate_box: list[float] | None = [10.0, 10.0, 30.0, 30.0]
review_box: list[float] | None = [10.0, 10.0, 30.0, 30.0]
if sequence == 2:
review_box = [8.0, 8.0, 40.0, 40.0]
elif sequence == 3:
review_box = [8.0, 8.0, 40.0, 40.0]
review_category = "person"
elif sequence == 4:
review_category = "person"
elif sequence == 5:
review_box = None
elif sequence == 6:
candidate_box = None
candidates.append(
_candidate_case(
sequence,
category=candidate_category,
box=candidate_box,
)
)
reviews.append(
_review_frame(
sequence,
category=review_category,
box=review_box,
)
)
return tuple(candidates), tuple(reviews)
def test_evaluation_separates_localization_from_unmatched_objects() -> None:
candidates, reviews = _fixture()
cases, metrics = evaluate_l34e_self_review_diagnostic(
l34d_cases=candidates,
annotation_frames=reviews,
)
diagnostic = metrics["diagnostic_association"]
assert diagnostic == {
"prediction_count": 31,
"reference_count": 31,
"associated_pair_count": 30,
"strict_alignment": 27,
"strict_class_mismatch": 1,
"localization_disagreement": 1,
"class_and_localization_disagreement": 1,
"prediction_only": 1,
"reference_only": 1,
"candidate_association_coverage": 30 / 31,
"reference_association_coverage": 30 / 31,
"error_case_count": 5,
}
strict = metrics["strict_iou50"]
assert strict["true_positive"] == 27
assert strict["false_positive"] == 4
assert strict["false_negative"] == 4
assert strict["class_mismatch"] == 1
assert cases[1]["predictions"][0]["diagnostic_verdict"] == (
"localization_disagreement"
)
assert cases[2]["references"][0]["diagnostic_verdict"] == (
"class_and_localization_disagreement"
)
assert cases[4]["predictions"][0]["diagnostic_verdict"] == "prediction_only"
assert cases[5]["references"][0]["diagnostic_verdict"] == "reference_only"
def test_evaluation_rejects_incomplete_coverage() -> None:
candidates, reviews = _fixture()
with pytest.raises(
L34ESelfReviewDiagnosticError,
match="exactly 32",
):
evaluate_l34e_self_review_diagnostic(
l34d_cases=candidates[:-1],
annotation_frames=reviews,
)
@@ -0,0 +1,111 @@
from __future__ import annotations
from typing import Any
import pytest
from PIL import Image
from k1link.compute.e49_detector_truth_evaluation import (
evaluate_frozen_detector_candidates,
)
from k1link.compute.l34_right_yolox_truth_island_freeze import (
L34_PREDICTION_SCHEMA,
)
from k1link.compute.l35_right_yolox_truth_evaluation import (
L35RightYoloxTruthEvaluationError,
_require_freeze_before_truth_seal,
l34_rows_for_sealed_truth,
)
def _l34_row() -> dict[str, Any]:
return {
"schema_version": L34_PREDICTION_SCHEMA,
"candidate_id": "yolox-s-kb4-core3",
"truth_island_sequence": 1,
"image_id": 11,
"frame_index": 70,
"session_seconds": 42.5,
"source_image_sha256": "a" * 64,
"group_id": "anchor-011",
"predictions": [
{
"label": "car",
"score": 0.9,
"bbox_xyxy": [10.0, 10.0, 110.0, 110.0],
}
],
"truth_joined": False,
}
def _truth_row() -> dict[str, Any]:
return {
"truth_island_sequence": 1,
"image_id": 11,
"frame_index": 70,
"session_seconds": 42.5,
"role": "anchor",
"group_id": "anchor-011",
"source_image_sha256": "a" * 64,
"hard_negative": False,
"objects": [
{
"object_id": "car-1",
"category": "car",
"box_xyxy": [10.0, 10.0, 110.0, 110.0],
"occluded": False,
"truncated": False,
"notes": None,
}
],
"adjudicated": True,
}
def test_l35_adapts_l34_without_changing_identity_or_boxes() -> None:
rows = l34_rows_for_sealed_truth((_l34_row(),))
assert rows == (
{
"candidate_id": "yolox-s-kb4-core3",
"truth_island_sequence": 1,
"image_id": 11,
"frame_index": 70,
"session_seconds": 42.5,
"source_image_sha256": "a" * 64,
"predictions": [
{
"category": "car",
"score": 0.9,
"box_xyxy": [10.0, 10.0, 110.0, 110.0],
}
],
"truth_joined": False,
},
)
metrics = evaluate_frozen_detector_candidates(
truth_rows=(_truth_row(),),
prediction_rows=rows,
valid_fov_mask=Image.new("L", (800, 600), color=255),
)
assert metrics["yolox-s-kb4-core3"]["ap50"] == 1.0
assert metrics["yolox-s-kb4-core3"]["candidate_winner_selected"] is False
def test_l35_rejects_a_freeze_created_after_truth_was_sealed() -> None:
with pytest.raises(
L35RightYoloxTruthEvaluationError,
match="postdates",
):
_require_freeze_before_truth_seal(
freeze_created_at_utc="2026-08-01T02:00:00Z",
truth_sealed_at_utc="2026-08-01T01:00:00Z",
)
def test_l35_accepts_a_freeze_created_before_truth_was_sealed() -> None:
_require_freeze_before_truth_seal(
freeze_created_at_utc="2026-08-01T00:30:00Z",
truth_sealed_at_utc="2026-08-01T01:00:00Z",
)
+13
View File
@@ -76,6 +76,9 @@ def test_camera_semantic_and_connected_occupied_support_agree() -> None:
)
assert support.document["geometry_status"] == "agree"
assert support.document["range_m"] == pytest.approx(2.0666666667)
assert support.document["range_estimate_m"] == pytest.approx(2.0666666667)
assert support.document["range_estimate_available"] is True
assert support.document["range_support_qualified"] is True
assert support.document["unknown_is_occupied"] is True
assert support.document["navigation_or_safety_accepted"] is False
assert support.occupied_source_indices.tolist() == [0, 1]
@@ -120,8 +123,18 @@ def test_camera_only_and_surface_conflict_remain_explicit() -> None:
profile=CameraGeometryFusionProfile(),
)
assert camera_only.document["geometry_status"] == "single-source-camera"
assert camera_only.document["range_m"] is None
assert camera_only.document["range_estimate_m"] == pytest.approx(2.0)
assert camera_only.document["range_estimate_available"] is True
assert camera_only.document["range_support_qualified"] is False
assert conflict.document["geometry_status"] == "conflict"
assert conflict.document["range_m"] is None
assert conflict.document["range_estimate_available"] is False
assert conflict.document["range_support_qualified"] is False
assert unknown.document["geometry_status"] == "unknown"
assert unknown.document["range_m"] is None
assert unknown.document["range_estimate_available"] is False
assert unknown.document["range_support_qualified"] is False
def test_unclaimed_occupied_component_is_a_separate_geometry_layer() -> None: