feat(perception): add recorded replay maturation labs
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import APIRouter
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
import k1link.compute.e46a_ai_engineering_preannotation as compute
|
||||
from k1link.web.e46a_ai_engineering_preannotation_api import (
|
||||
build_e46a_ai_engineering_preannotation_router,
|
||||
)
|
||||
from k1link.web.l34_annotation_api import (
|
||||
L34AnnotationCreateRequest,
|
||||
L34AnnotationFrameRequest,
|
||||
L34AnnotationObjectRequest,
|
||||
L34AnnotationSaveRequest,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint(router: APIRouter, path: str, method: str = "GET") -> object:
|
||||
for route in router.routes:
|
||||
if (
|
||||
isinstance(route, APIRoute)
|
||||
and route.path == path
|
||||
and route.methods is not None
|
||||
and method in route.methods
|
||||
):
|
||||
return route.endpoint
|
||||
raise AssertionError(f"{method} {path} route is missing")
|
||||
|
||||
|
||||
def _build_fixture(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> dict[str, object]:
|
||||
e46_id = f"e46-detector-truth-island-{'a' * 64}"
|
||||
e46_root = tmp_path / "e46" / e46_id
|
||||
e46_root.mkdir(parents=True)
|
||||
(e46_root / "manifest.json").write_text("{}\n", encoding="utf-8")
|
||||
references = []
|
||||
cases = []
|
||||
for sequence in range(1, 33):
|
||||
source_sha = f"{sequence:064x}"
|
||||
references.append(
|
||||
{
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence * 100,
|
||||
"group_id": f"group-{sequence:02d}",
|
||||
"sha256": source_sha,
|
||||
}
|
||||
)
|
||||
objects = [
|
||||
{
|
||||
"object_id": f"object-{sequence:02d}-car",
|
||||
"category": "car",
|
||||
"proposed_label": None,
|
||||
"origin": "self_review_seed",
|
||||
"box_xyxy": [10.0, 20.0, 100.0, 120.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
}
|
||||
]
|
||||
if sequence in {5, 29}:
|
||||
objects.append(
|
||||
{
|
||||
"object_id": f"object-{sequence:02d}-custom",
|
||||
"category": "unmapped",
|
||||
"proposed_label": (
|
||||
"Детская коляска" if sequence == 5 else "Ноутбук"
|
||||
),
|
||||
"origin": "self_review_seed",
|
||||
"box_xyxy": [120.0, 140.0, 280.0, 320.0],
|
||||
"occluded": False,
|
||||
"truncated": sequence == 29,
|
||||
}
|
||||
)
|
||||
cases.append(
|
||||
{
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence * 100,
|
||||
"group_id": f"group-{sequence:02d}",
|
||||
"session_seconds": float(sequence),
|
||||
"source_image_sha256": source_sha,
|
||||
"references": objects,
|
||||
}
|
||||
)
|
||||
(e46_root / "image-references.jsonl").write_text(
|
||||
"".join(json.dumps(row) + "\n" for row in references),
|
||||
encoding="utf-8",
|
||||
)
|
||||
l34f_id = f"l34f-adjudicated-reference-{'b' * 64}"
|
||||
l34f_root = tmp_path / "l34f" / l34f_id
|
||||
l34f_root.mkdir(parents=True)
|
||||
(l34f_root / "manifest.json").write_text("{}\n", encoding="utf-8")
|
||||
e46 = SimpleNamespace(
|
||||
result_id=e46_id,
|
||||
result_root=e46_root,
|
||||
manifest={},
|
||||
report={},
|
||||
)
|
||||
l34f = {
|
||||
"result_id": l34f_id,
|
||||
"result_root": l34f_root,
|
||||
"manifest": {},
|
||||
"report": {},
|
||||
"cases": tuple(cases),
|
||||
}
|
||||
monkeypatch.setattr(compute, "read_e46_detector_truth_island", lambda _: e46)
|
||||
monkeypatch.setattr(compute, "read_l34f_adjudicated_reference", lambda _: l34f)
|
||||
return compute.build_e46a_ai_engineering_preannotation(
|
||||
e46_root=e46_root,
|
||||
l34f_root=l34f_root,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
|
||||
|
||||
def test_e46a_freezes_ai_preannotation_without_truth_authority(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result = _build_fixture(tmp_path, monkeypatch)
|
||||
|
||||
assert result["report"]["metrics"]["frame_count"] == 32
|
||||
assert result["report"]["metrics"]["object_count"] == 34
|
||||
assert result["report"]["metrics"]["custom_class_relabel_count"] == 2
|
||||
assert result["report"]["metrics"]["independent_review_submission_count"] == 0
|
||||
assert result["report"]["decision"]["e48_truth_seal_open"] is False
|
||||
assert result["manifest"]["authority"]["independent_truth"] is False
|
||||
custom = {
|
||||
item["category"]
|
||||
for case in result["cases"]
|
||||
for item in case["objects"]
|
||||
if item["category"] in {"stroller", "laptop"}
|
||||
}
|
||||
assert custom == {"stroller", "laptop"}
|
||||
assert all(
|
||||
item["category"] != "unmapped"
|
||||
for case in result["cases"]
|
||||
for item in case["objects"]
|
||||
)
|
||||
|
||||
|
||||
def test_e46a_object_level_audit_deletes_snaps_and_relabels(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result = _build_fixture(tmp_path, monkeypatch)
|
||||
cases = copy.deepcopy(list(result["cases"]))
|
||||
rows = tuple(
|
||||
{
|
||||
"candidate_id": "geometry-candidate",
|
||||
"truth_island_sequence": sequence,
|
||||
"source_image_sha256": cases[sequence - 1][
|
||||
"source_image_sha256"
|
||||
],
|
||||
"predictions": [
|
||||
{
|
||||
"category": "car",
|
||||
"score": 0.95,
|
||||
"box_xyxy": [12.0, 22.0, 98.0, 118.0],
|
||||
}
|
||||
],
|
||||
}
|
||||
for sequence in range(1, 33)
|
||||
)
|
||||
profile = compute.E46AVisualAuditProfile(
|
||||
profile_id="fixture-object-qa/v1",
|
||||
geometry_candidate_id="geometry-candidate",
|
||||
candidate_nms_iou=0.3,
|
||||
maximum_match_cost=1.3,
|
||||
expected_source_object_count=34,
|
||||
expected_final_object_count=33,
|
||||
expected_geometry_snapped_count=31,
|
||||
delete_object_ids=("object-01-car",),
|
||||
category_overrides=(("object-02-car", "heavy_vehicle"),),
|
||||
)
|
||||
|
||||
audit = compute._apply_visual_audit( # noqa: SLF001
|
||||
cases=cases,
|
||||
prediction_rows=rows,
|
||||
profile=profile,
|
||||
)
|
||||
|
||||
assert audit["deleted_false_box_count"] == 1
|
||||
assert audit["geometry_snapped_object_count"] == 31
|
||||
assert audit["source_geometry_retained_object_count"] == 2
|
||||
assert cases[0]["objects"] == []
|
||||
assert cases[0]["hard_negative"] is True
|
||||
assert cases[1]["objects"][0]["category"] == "heavy_vehicle"
|
||||
assert cases[1]["objects"][0]["box_xyxy"] == [12.0, 22.0, 98.0, 118.0]
|
||||
|
||||
|
||||
def test_e46a_api_projects_visual_cases_without_truth_escalation(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result = _build_fixture(tmp_path, monkeypatch)
|
||||
router = build_e46a_ai_engineering_preannotation_router(
|
||||
root_provider=lambda: tmp_path / "results"
|
||||
)
|
||||
catalog = _endpoint(router, "/api/v1/laboratory/e46a/results")(limit=1) # type: ignore[operator]
|
||||
assert catalog["items"][0]["result_id"] == result["result_id"]
|
||||
assert catalog["items"][0]["ground_truth"] is False
|
||||
|
||||
case = _endpoint(
|
||||
router,
|
||||
"/api/v1/laboratory/e46a/results/{result_id}/cases/{sequence}",
|
||||
)(result_id=result["result_id"], sequence=5) # type: ignore[operator]
|
||||
assert case["independent_review"] is False
|
||||
assert case["ground_truth"] is False
|
||||
assert any(item["category"] == "stroller" for item in case["objects"])
|
||||
assert case["camera_url"].endswith("/annotation-source/frames/5/camera")
|
||||
|
||||
|
||||
def test_e46a_correction_session_starts_from_editable_ai_seed(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result = _build_fixture(tmp_path, monkeypatch)
|
||||
router = build_e46a_ai_engineering_preannotation_router(
|
||||
root_provider=lambda: tmp_path / "results",
|
||||
annotation_root_provider=lambda: tmp_path / "annotations",
|
||||
)
|
||||
result_id = str(result["result_id"])
|
||||
|
||||
source = _endpoint(
|
||||
router,
|
||||
"/api/v1/laboratory/e46a/results/{result_id}/annotation-source",
|
||||
)(result_id=result_id) # type: ignore[operator]
|
||||
assert source["frame_count"] == 32
|
||||
assert source["contract"]["contract_id"] == (
|
||||
"e46a-ai-engineering-correction/v1"
|
||||
)
|
||||
assert source["candidate_identity_included"] is True
|
||||
assert source["candidate_predictions_included"] is False
|
||||
assert source["prelabels_included"] is False
|
||||
|
||||
seed = _endpoint(
|
||||
router,
|
||||
(
|
||||
"/api/v1/laboratory/e46a/results/{result_id}"
|
||||
"/annotation-seed/frames/{sequence}"
|
||||
),
|
||||
)(result_id=result_id, sequence=5) # type: ignore[operator]
|
||||
custom = next(
|
||||
item for item in seed["objects"] if item["category"] == "unmapped"
|
||||
)
|
||||
assert custom["proposed_label"] == "Коляска"
|
||||
assert custom["origin"] == "frozen_candidate_seed"
|
||||
|
||||
create = _endpoint(
|
||||
router,
|
||||
"/api/v1/laboratory/e46a/results/{result_id}/annotation-sessions",
|
||||
"POST",
|
||||
)
|
||||
created = create( # type: ignore[operator]
|
||||
result_id=result_id,
|
||||
request=L34AnnotationCreateRequest(idempotency_key="e46a-correction-1"),
|
||||
)
|
||||
assert created["contract_id"] == "e46a-ai-engineering-correction/v1"
|
||||
assert created["revision"] == 0
|
||||
assert created["frames"] == []
|
||||
|
||||
save = _endpoint(
|
||||
router,
|
||||
(
|
||||
"/api/v1/laboratory/e46a/results/{result_id}"
|
||||
"/annotation-sessions/{session_id}"
|
||||
),
|
||||
"PUT",
|
||||
)
|
||||
saved = save( # type: ignore[operator]
|
||||
result_id=result_id,
|
||||
session_id=created["session_id"],
|
||||
request=L34AnnotationSaveRequest(
|
||||
expected_revision=0,
|
||||
idempotency_key="e46a-save-1",
|
||||
title="E46A human correction",
|
||||
assistance_mode="frozen-candidate-seeded",
|
||||
frames=[
|
||||
L34AnnotationFrameRequest(
|
||||
truth_island_sequence=5,
|
||||
reviewed=True,
|
||||
hard_negative=False,
|
||||
objects=[
|
||||
L34AnnotationObjectRequest(
|
||||
object_id=str(custom["object_id"]),
|
||||
category="unmapped",
|
||||
proposed_label="Коляска",
|
||||
origin="frozen_candidate_seed",
|
||||
box_xyxy=[130.0, 150.0, 300.0, 340.0],
|
||||
occluded=False,
|
||||
truncated=False,
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
assert saved["revision"] == 1
|
||||
assert saved["frames"][0]["objects"][0]["box_xyxy"] == [
|
||||
130.0,
|
||||
150.0,
|
||||
300.0,
|
||||
340.0,
|
||||
]
|
||||
assert saved["authority"]["ground_truth"] is False
|
||||
|
||||
|
||||
def test_e46a_reader_rejects_tampered_cases(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
result = _build_fixture(tmp_path, monkeypatch)
|
||||
cases_path = Path(result["result_root"]) / compute.E46A_CASES_NAME
|
||||
cases_path.write_text(cases_path.read_text(encoding="utf-8") + "{}\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(compute.E46AAiEngineeringPreannotationError):
|
||||
compute.read_e46a_ai_engineering_preannotation(Path(result["result_root"]))
|
||||
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from k1link.compute.e46d_temporal_failure_audit import analyze_temporal_frames
|
||||
|
||||
|
||||
def _object(
|
||||
track_id: int,
|
||||
*,
|
||||
box: list[float] | None = None,
|
||||
camera_current: bool = True,
|
||||
world_track_id: int | None = 240001,
|
||||
motion_state: str = "static",
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"bbox_xyxy": box or [100.0, 100.0, 180.0, 220.0],
|
||||
"category": "car",
|
||||
"route_track_id": track_id,
|
||||
"world_track_id": world_track_id,
|
||||
"motion_state": motion_state,
|
||||
"camera_evidence_current": camera_current,
|
||||
}
|
||||
|
||||
|
||||
def _frames() -> list[dict[str, object]]:
|
||||
rows: list[dict[str, object]] = []
|
||||
for frame_index in range(30):
|
||||
rows.append(
|
||||
{
|
||||
"frame_index": frame_index,
|
||||
"session_seconds": 35.0 + frame_index * 0.1,
|
||||
"objects": [],
|
||||
}
|
||||
)
|
||||
for frame_index in (0, 1, 2, 6, 7, 8):
|
||||
rows[frame_index]["objects"] = [_object(10)]
|
||||
rows[3]["objects"] = []
|
||||
rows[4]["objects"] = []
|
||||
rows[5]["objects"] = []
|
||||
rows[10]["objects"] = [_object(20)]
|
||||
rows[11]["objects"] = [_object(21)]
|
||||
rows[12]["objects"] = [_object(30, box=[100.0, 100.0, 180.0, 220.0])]
|
||||
rows[13]["objects"] = [_object(30, box=[300.0, 100.0, 380.0, 220.0])]
|
||||
for offset, state in enumerate(("static", "unknown", "static", "unknown", "static")):
|
||||
rows[15 + offset]["objects"] = [
|
||||
_object(40, motion_state=state, world_track_id=240001 + offset % 2)
|
||||
]
|
||||
for frame_index in range(20, 25):
|
||||
rows[frame_index]["objects"] = [_object(50, camera_current=frame_index in {20, 24})]
|
||||
for offset, track_id in enumerate((60, 61, 62, 63)):
|
||||
rows[26 + offset]["objects"] = [_object(track_id)]
|
||||
return rows
|
||||
|
||||
|
||||
def test_e46d_scans_observable_temporal_failures_and_ranks_video_clips() -> None:
|
||||
signals, clips, metrics = analyze_temporal_frames(_frames())
|
||||
kinds = {signal["kind"] for signal in signals}
|
||||
|
||||
assert "layer-blackout" in kinds
|
||||
assert "route-layer-gap" in kinds
|
||||
assert "route-id-rebirth-candidate" in kinds
|
||||
assert "bbox-jump" in kinds
|
||||
assert "motion-state-flap" in kinds
|
||||
assert "world-binding-flap" in kinds
|
||||
assert "short-track-burst" in kinds
|
||||
assert metrics["temporal_continuity_passed"] is False
|
||||
assert metrics["failure_signal_count"] == len(signals)
|
||||
assert metrics["review_clip_count"] == len(clips)
|
||||
assert clips[0]["priority"] == "critical"
|
||||
assert all(clip["start_seconds"] <= clip["event_start_seconds"] for clip in clips)
|
||||
assert all(clip["event_end_seconds"] <= clip["end_seconds"] for clip in clips)
|
||||
|
||||
|
||||
def test_e46d_distinguishes_camera_hold_from_route_layer_loss() -> None:
|
||||
frames = _frames()
|
||||
signals, _, metrics = analyze_temporal_frames(frames)
|
||||
|
||||
held = [signal for signal in signals if signal["kind"] == "camera-evidence-hold"]
|
||||
gaps = [signal for signal in signals if signal["kind"] == "route-layer-gap"]
|
||||
assert held
|
||||
assert gaps
|
||||
assert all(signal["evidence"]["camera_evidence_current"] is False for signal in held)
|
||||
assert all(signal["evidence"]["same_route_identity_returned"] is True for signal in gaps)
|
||||
assert metrics["detector_hold_episode_count"] == len(held)
|
||||
@@ -0,0 +1,279 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e46e_ready_stack import (
|
||||
E46E_RUNTIME_SCHEMA,
|
||||
E46EReadyStackError,
|
||||
analyze_e46e_frames,
|
||||
build_e46e_ready_stack,
|
||||
read_e46e_ready_stack,
|
||||
)
|
||||
|
||||
|
||||
def test_e46e_projects_stock_deepstream_output_without_custom_tracking(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source, profile = _source_and_profile(tmp_path, frame_count=6)
|
||||
raw = _raw_output(tmp_path, profile, frame_count=6)
|
||||
_detector(raw, 0, "car", (10, 20, 40, 60), 0.91)
|
||||
_tracker(raw, 0, 7, "car", (10, 20, 40, 60), 0.88)
|
||||
_tracker(raw, 1, 7, "car", (12, 20, 42, 60), 0.73)
|
||||
_detector(raw, 5, "person", (100, 80, 130, 160), 0.93)
|
||||
_tracker(raw, 5, 7, "person", (100, 80, 130, 160), 0.79)
|
||||
|
||||
result = build_e46e_ready_stack(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
metrics = result["report"]["metrics"]
|
||||
assert metrics["frame_count"] == 6
|
||||
assert metrics["detection_observation_count"] == 2
|
||||
assert metrics["track_observation_count"] == 3
|
||||
assert metrics["detection_box_clipped_count"] == 0
|
||||
assert metrics["track_box_clipped_count"] == 0
|
||||
assert metrics["tracker_recovered_frame_count"] == 1
|
||||
assert metrics["full_layer_blackout_event_count"] == 1
|
||||
assert metrics["route_id_gap_event_count"] == 1
|
||||
assert metrics["track_class_switch_count"] == 1
|
||||
assert result["frames"][1]["objects"][0]["object_id"] == "nvdcf-7"
|
||||
assert result["frames"][1]["objects"][0]["bbox"] == [12.0, 20.0, 30.0, 40.0]
|
||||
assert [
|
||||
(component["kind"], component["role"])
|
||||
for component in result["report"]["method"]["components"]
|
||||
] == [
|
||||
("source", "exact recorded camera evidence"),
|
||||
("model", "framewise traffic-object detection"),
|
||||
("tool", "official RT-DETR output decoding"),
|
||||
("algorithm", "route-local temporal association"),
|
||||
("runtime", "GPU inference and media pipeline"),
|
||||
]
|
||||
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
|
||||
assert result["manifest"]["authority"]["navigation_or_safety_accepted"] is False
|
||||
|
||||
repeated = build_e46e_ready_stack(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
assert repeated["result_id"] == result["result_id"]
|
||||
|
||||
|
||||
def test_e46e_clips_only_display_geometry_at_the_source_plane(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path, frame_count=1)
|
||||
raw = _raw_output(tmp_path, profile, frame_count=1)
|
||||
_tracker(raw, 0, 31, "person", (447, 547, 527, 608), 0.69)
|
||||
|
||||
result = build_e46e_ready_stack(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
|
||||
item = result["frames"][0]["objects"][0]
|
||||
assert item["bbox"] == [447.0, 547.0, 80.0, 53.0]
|
||||
assert item["source_bbox_ltrb"] == [447.0, 547.0, 527.0, 608.0]
|
||||
assert item["source_plane_clipped"] is True
|
||||
assert result["report"]["metrics"]["track_box_clipped_count"] == 1
|
||||
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
|
||||
|
||||
|
||||
def test_e46e_rejects_tampered_immutable_artifact(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path, frame_count=1)
|
||||
raw = _raw_output(tmp_path, profile, frame_count=1)
|
||||
result = build_e46e_ready_stack(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
(result["result_root"] / "overlay.mp4").write_bytes(b"changed")
|
||||
with pytest.raises(E46EReadyStackError, match="artifact changed"):
|
||||
read_e46e_ready_stack(result["result_root"])
|
||||
|
||||
|
||||
def test_e46e_frame_analyzer_requires_contiguous_source_order() -> None:
|
||||
with pytest.raises(E46EReadyStackError, match="not contiguous"):
|
||||
analyze_e46e_frames(
|
||||
[
|
||||
{
|
||||
"frame_index": 1,
|
||||
"session_seconds": 1.0,
|
||||
"detections": [],
|
||||
"objects": [],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _source_and_profile(tmp_path: Path, *, frame_count: int) -> tuple[Path, Path]:
|
||||
source = tmp_path / "source-job"
|
||||
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
|
||||
camera.mkdir(parents=True)
|
||||
index_rows = [
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording-index/v1",
|
||||
"kind": "media",
|
||||
"sequence": sequence,
|
||||
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
|
||||
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
|
||||
}
|
||||
for sequence in range(1, frame_count + 1)
|
||||
]
|
||||
index_path = camera / "index.jsonl"
|
||||
index_path.write_text(
|
||||
"".join(json.dumps(row, sort_keys=True) + "\n" for row in index_rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stream_sha = "1" * 64
|
||||
summary_path = camera / "summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording/v1",
|
||||
"stream_sha256": stream_sha,
|
||||
"segment_count": frame_count,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
index_sha = _sha(index_path)
|
||||
summary_sha = _sha(summary_path)
|
||||
job = {
|
||||
"schema_version": "missioncore.compute-job/v1",
|
||||
"job_id": "recorded-test",
|
||||
"input": {
|
||||
"session_id": "test-session",
|
||||
"source_id": "sensor.camera.right",
|
||||
"segment_count": frame_count,
|
||||
"archive_index_sha256": index_sha,
|
||||
"archive_summary_sha256": summary_sha,
|
||||
"timeline": {"start_seconds": 10.0, "end_seconds": 11.0},
|
||||
},
|
||||
}
|
||||
(source / "job.json").write_text(json.dumps(job, indent=2) + "\n", encoding="utf-8")
|
||||
profile_value = {
|
||||
"schema_version": "missioncore.e46e-ready-stack-profile/v1",
|
||||
"profile_id": "e46e-test/v1",
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-test",
|
||||
"session_id": "test-session",
|
||||
"segment_count": frame_count,
|
||||
"stream_sha256": stream_sha,
|
||||
"archive_index_sha256": index_sha,
|
||||
"archive_summary_sha256": summary_sha,
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:"
|
||||
+ "2" * 64,
|
||||
"deepstream_version": "9.1",
|
||||
},
|
||||
"detector": {
|
||||
"name": "NVIDIA TrafficCamNet Transformer Lite",
|
||||
"version": "test",
|
||||
"model_sha256": "3" * 64,
|
||||
"custom_postprocessing": False,
|
||||
},
|
||||
"parser": {
|
||||
"name": "NVIDIA DeepStream TAO custom bounding-box parser",
|
||||
"repository": "https://github.com/NVIDIA/DeepStream.git",
|
||||
"commit": "8" * 40,
|
||||
"symbol": "NvDsInferParseCustomDDETRTAO",
|
||||
"library_sha256": "9" * 64,
|
||||
"custom_mission_core_logic": False,
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"configuration": "stock-accuracy",
|
||||
"custom_association": False,
|
||||
"custom_hold_or_stitch": False,
|
||||
},
|
||||
"output": {"frame_width": 800, "frame_height": 600},
|
||||
"authority": {
|
||||
"ground_truth": False,
|
||||
"candidate_accepted": False,
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
profile = tmp_path / "profile.json"
|
||||
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
|
||||
return source, profile
|
||||
|
||||
|
||||
def _raw_output(tmp_path: Path, profile_path: Path, *, frame_count: int) -> Path:
|
||||
raw = tmp_path / "raw"
|
||||
detections = raw / "detections"
|
||||
tracks = raw / "tracks"
|
||||
detections.mkdir(parents=True)
|
||||
tracks.mkdir()
|
||||
for frame in range(frame_count):
|
||||
(detections / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
|
||||
(tracks / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
|
||||
overlay = raw / "overlay.mp4"
|
||||
overlay.write_bytes(b"synthetic-overlay")
|
||||
(raw / "deepstream.log").write_text("synthetic success\n", encoding="utf-8")
|
||||
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||
image = profile["runtime"]["container_image"]
|
||||
runtime = {
|
||||
"schema_version": E46E_RUNTIME_SCHEMA,
|
||||
"status": "completed",
|
||||
"worker_host": "TEST-WORKER-006",
|
||||
"gpu_name": "Synthetic RTX",
|
||||
"container_image": image,
|
||||
"container_image_digest": image.rsplit("@sha256:", 1)[1],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
"model_engine_sha256": "4" * 64,
|
||||
"deepstream_config_sha256": "5" * 64,
|
||||
"detector_config_sha256": "6" * 64,
|
||||
"parser_library_sha256": profile["parser"]["library_sha256"],
|
||||
"tracker_config_sha256": "7" * 64,
|
||||
"input_stream_sha256": profile["source"]["stream_sha256"],
|
||||
"overlay_sha256": _sha(overlay),
|
||||
}
|
||||
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
|
||||
return raw
|
||||
|
||||
|
||||
def _detector(
|
||||
raw: Path,
|
||||
frame: int,
|
||||
label: str,
|
||||
box: tuple[int, int, int, int],
|
||||
confidence: float,
|
||||
) -> None:
|
||||
left, top, right, bottom = box
|
||||
(raw / "detections" / f"00_000_{frame:06d}.txt").write_text(
|
||||
f"{label} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _tracker(
|
||||
raw: Path,
|
||||
frame: int,
|
||||
track_id: int,
|
||||
label: str,
|
||||
box: tuple[int, int, int, int],
|
||||
confidence: float,
|
||||
) -> None:
|
||||
left, top, right, bottom = box
|
||||
(raw / "tracks" / f"00_000_{frame:06d}.txt").write_text(
|
||||
f"{label} {track_id} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _module() -> object:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "prepare_e46e_worker_package.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("e46e_worker_package_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_e46e_package_is_minimal_deterministic_and_content_addressed(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
profile, parser_library = _package_inputs(tmp_path, repository)
|
||||
kwargs = {
|
||||
"repository_root": repository,
|
||||
"profile_path": profile,
|
||||
"parser_library_path": parser_library,
|
||||
"output_root": tmp_path,
|
||||
}
|
||||
package = module.build_e46e_worker_package(**kwargs)
|
||||
repeated = module.build_e46e_worker_package(**kwargs)
|
||||
manifest = module.validate_e46e_worker_package(package)
|
||||
assert repeated == package
|
||||
assert package.name == f"e46e-worker-package-{manifest['identity_sha256']}"
|
||||
assert manifest["identity"]["classification"] == (
|
||||
"minimal-stock-nvidia-recorded-right-worker-package"
|
||||
)
|
||||
assert manifest["identity"]["tracker"]["custom_association"] is False
|
||||
assert manifest["identity"]["tracker"]["custom_hold_or_stitch"] is False
|
||||
assert manifest["identity"]["parser"]["custom_mission_core_logic"] is False
|
||||
assert len(manifest["artifacts"]) == 13
|
||||
|
||||
|
||||
def test_e46e_package_rejects_unexpected_member(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
profile, parser_library = _package_inputs(tmp_path, repository)
|
||||
package = module.build_e46e_worker_package(
|
||||
repository_root=repository,
|
||||
profile_path=profile,
|
||||
parser_library_path=parser_library,
|
||||
output_root=tmp_path,
|
||||
)
|
||||
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
|
||||
with pytest.raises(module.E46EWorkerPackageError, match="file set changed"):
|
||||
module.validate_e46e_worker_package(package)
|
||||
|
||||
|
||||
def _package_inputs(tmp_path: Path, repository: Path) -> tuple[Path, Path]:
|
||||
parser_library = tmp_path / "libnvds_infercustomparser_tao.so"
|
||||
parser_library.write_bytes(b"synthetic official parser fixture")
|
||||
profile_value = json.loads(
|
||||
(
|
||||
repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46e_ready_stack_profile.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
profile_value["parser"]["library_sha256"] = hashlib.sha256(
|
||||
parser_library.read_bytes()
|
||||
).hexdigest()
|
||||
profile = tmp_path / "profile.json"
|
||||
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
|
||||
return profile, parser_library
|
||||
@@ -0,0 +1,227 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e46f_dashcam_bakeoff import (
|
||||
E46F_RUNTIME_SCHEMA,
|
||||
E46FDashCamBakeoffError,
|
||||
build_e46f_dashcam_bakeoff,
|
||||
read_e46f_dashcam_bakeoff,
|
||||
)
|
||||
|
||||
|
||||
def test_e46f_freezes_only_the_stock_detector_change(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path, frame_count=3)
|
||||
raw = _raw_output(tmp_path, profile, frame_count=3)
|
||||
_observation(raw, "detections", 0, "car", (10, 20, 40, 60), 0.91)
|
||||
_observation(raw, "tracks", 0, "car", (10, 20, 40, 60), 0.88, track_id=7)
|
||||
_observation(raw, "tracks", 1, "car", (12, 20, 42, 60), 0.73, track_id=7)
|
||||
_observation(raw, "detections", 2, "person", (100, 80, 130, 160), 0.93)
|
||||
_observation(raw, "tracks", 2, "person", (100, 80, 130, 160), 0.79, track_id=8)
|
||||
|
||||
result = build_e46f_dashcam_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
|
||||
metrics = result["report"]["metrics"]
|
||||
assert metrics["frame_count"] == 3
|
||||
assert metrics["detection_observation_count"] == 2
|
||||
assert metrics["track_observation_count"] == 3
|
||||
assert metrics["tracker_recovered_frame_count"] == 1
|
||||
assert result["frames"][0]["detections"][0]["provenance"] == ("nvidia-dashcamnet-detectnet-v2")
|
||||
assert result["report"]["comparison_contract"] == {
|
||||
"baseline_result_id": f"e46e-ready-stack-{'a' * 64}",
|
||||
"controlled_change": "detector-only",
|
||||
"held_constant": ["recorded RIGHT source", "DeepStream", "FP16", "NvDCF"],
|
||||
}
|
||||
assert result["report"]["decision"]["custom_temporal_logic_used"] is False
|
||||
assert result["manifest"]["authority"]["navigation_or_safety_accepted"] is False
|
||||
|
||||
repeated = build_e46f_dashcam_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
assert repeated["result_id"] == result["result_id"]
|
||||
|
||||
|
||||
def test_e46f_rejects_tampered_immutable_video(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path, frame_count=1)
|
||||
raw = _raw_output(tmp_path, profile, frame_count=1)
|
||||
result = build_e46f_dashcam_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
(result["result_root"] / "overlay.mp4").write_bytes(b"changed")
|
||||
with pytest.raises(E46FDashCamBakeoffError, match="artifact changed"):
|
||||
read_e46f_dashcam_bakeoff(result["result_root"])
|
||||
|
||||
|
||||
def _source_and_profile(tmp_path: Path, *, frame_count: int) -> tuple[Path, Path]:
|
||||
source = tmp_path / "source-job"
|
||||
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
|
||||
camera.mkdir(parents=True)
|
||||
rows = [
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording-index/v1",
|
||||
"kind": "media",
|
||||
"sequence": sequence,
|
||||
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
|
||||
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
|
||||
}
|
||||
for sequence in range(1, frame_count + 1)
|
||||
]
|
||||
index_path = camera / "index.jsonl"
|
||||
index_path.write_text(
|
||||
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stream_sha = "1" * 64
|
||||
summary_path = camera / "summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording/v1",
|
||||
"stream_sha256": stream_sha,
|
||||
"segment_count": frame_count,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
index_sha = _sha(index_path)
|
||||
summary_sha = _sha(summary_path)
|
||||
(source / "job.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.compute-job/v1",
|
||||
"job_id": "recorded-test",
|
||||
"input": {
|
||||
"session_id": "test-session",
|
||||
"source_id": "sensor.camera.right",
|
||||
"segment_count": frame_count,
|
||||
"archive_index_sha256": index_sha,
|
||||
"archive_summary_sha256": summary_sha,
|
||||
"timeline": {"start_seconds": 10.0, "end_seconds": 11.0},
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
profile_value = {
|
||||
"schema_version": "missioncore.e46f-dashcam-bakeoff-profile/v1",
|
||||
"profile_id": "e46f-test/v1",
|
||||
"comparison_contract": {
|
||||
"baseline_result_id": f"e46e-ready-stack-{'a' * 64}",
|
||||
"controlled_change": "detector-only",
|
||||
"held_constant": ["recorded RIGHT source", "DeepStream", "FP16", "NvDCF"],
|
||||
},
|
||||
"source": {
|
||||
"camera_source_id": "sensor.camera.right",
|
||||
"job_id": "recorded-test",
|
||||
"session_id": "test-session",
|
||||
"segment_count": frame_count,
|
||||
"stream_sha256": stream_sha,
|
||||
"archive_index_sha256": index_sha,
|
||||
"archive_summary_sha256": summary_sha,
|
||||
},
|
||||
"runtime": {
|
||||
"container_image": "nvcr.io/nvidia/deepstream:9.1-samples-multiarch@sha256:" + "2" * 64,
|
||||
"deepstream_version": "9.1",
|
||||
},
|
||||
"detector": {
|
||||
"name": "NVIDIA DashCamNet",
|
||||
"version": "pruned_onnx_v1.0.4",
|
||||
"model_sha256": "3" * 64,
|
||||
"custom_postprocessing": False,
|
||||
},
|
||||
"postprocessor": {
|
||||
"name": "NVIDIA DeepStream built-in DetectNet_v2 parser and NMS",
|
||||
"cluster_mode": "NMS",
|
||||
"reference_commit": "4" * 40,
|
||||
"reference_config_sha256": "5" * 64,
|
||||
"custom_mission_core_logic": False,
|
||||
},
|
||||
"tracker": {
|
||||
"name": "NVIDIA NvDCF",
|
||||
"configuration": "stock-performance",
|
||||
"custom_association": False,
|
||||
"custom_hold_or_stitch": False,
|
||||
},
|
||||
"output": {"frame_width": 800, "frame_height": 600},
|
||||
"authority": {
|
||||
"ground_truth": False,
|
||||
"candidate_accepted": False,
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
},
|
||||
}
|
||||
profile = tmp_path / "profile.json"
|
||||
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
|
||||
return source, profile
|
||||
|
||||
|
||||
def _raw_output(tmp_path: Path, profile_path: Path, *, frame_count: int) -> Path:
|
||||
raw = tmp_path / "raw"
|
||||
for name in ("detections", "tracks"):
|
||||
directory = raw / name
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
for frame in range(frame_count):
|
||||
(directory / f"00_000_{frame:06d}.txt").write_text("", encoding="utf-8")
|
||||
overlay = raw / "overlay.mp4"
|
||||
overlay.write_bytes(b"synthetic-overlay")
|
||||
(raw / "deepstream.log").write_text("synthetic success\n", encoding="utf-8")
|
||||
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||
image = profile["runtime"]["container_image"]
|
||||
runtime = {
|
||||
"schema_version": E46F_RUNTIME_SCHEMA,
|
||||
"status": "completed",
|
||||
"worker_host": "TEST-WORKER-006",
|
||||
"gpu_name": "Synthetic RTX",
|
||||
"container_image": image,
|
||||
"container_image_digest": image.rsplit("@sha256:", 1)[1],
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
"model_engine_sha256": "6" * 64,
|
||||
"deepstream_config_sha256": "7" * 64,
|
||||
"detector_config_sha256": "8" * 64,
|
||||
"tracker_config_sha256": "9" * 64,
|
||||
"input_stream_sha256": profile["source"]["stream_sha256"],
|
||||
"overlay_sha256": _sha(overlay),
|
||||
}
|
||||
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
|
||||
return raw
|
||||
|
||||
|
||||
def _observation(
|
||||
raw: Path,
|
||||
directory: str,
|
||||
frame: int,
|
||||
label: str,
|
||||
box: tuple[int, int, int, int],
|
||||
confidence: float,
|
||||
*,
|
||||
track_id: int | None = None,
|
||||
) -> None:
|
||||
left, top, right, bottom = box
|
||||
identity = "" if track_id is None else f" {track_id}"
|
||||
(raw / directory / f"00_000_{frame:06d}.txt").write_text(
|
||||
f"{label}{identity} 0.0 0 0.0 {left} {top} {right} {bottom} 0 0 0 0 0 0 0 {confidence}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _module() -> object:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "prepare_e46f_worker_package.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("e46f_worker_package_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_e46f_package_is_minimal_deterministic_and_detector_only(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
kwargs = {
|
||||
"repository_root": repository,
|
||||
"profile_path": repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46f_dashcam_bakeoff_profile.json",
|
||||
"output_root": tmp_path,
|
||||
}
|
||||
package = module.build_e46f_worker_package(**kwargs)
|
||||
repeated = module.build_e46f_worker_package(**kwargs)
|
||||
manifest = module.validate_e46f_worker_package(package)
|
||||
assert repeated == package
|
||||
assert package.name == f"e46f-worker-package-{manifest['identity_sha256']}"
|
||||
assert manifest["identity"]["classification"] == (
|
||||
"minimal-stock-nvidia-detector-only-bakeoff-package"
|
||||
)
|
||||
assert manifest["identity"]["baseline_result_id"].startswith("e46e-ready-stack-")
|
||||
assert manifest["identity"]["tracker"]["custom_association"] is False
|
||||
assert manifest["identity"]["tracker"]["custom_hold_or_stitch"] is False
|
||||
assert manifest["identity"]["postprocessor"]["custom_mission_core_logic"] is False
|
||||
assert len(manifest["artifacts"]) == 11
|
||||
|
||||
|
||||
def test_e46f_package_rejects_unexpected_member(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
package = module.build_e46f_worker_package(
|
||||
repository_root=repository,
|
||||
profile_path=repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46f_dashcam_bakeoff_profile.json",
|
||||
output_root=tmp_path,
|
||||
)
|
||||
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
|
||||
with pytest.raises(module.E46FWorkerPackageError, match="file set changed"):
|
||||
module.validate_e46f_worker_package(package)
|
||||
@@ -0,0 +1,258 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e46g_rectified_detector_bakeoff import (
|
||||
E46G_RUNTIME_SCHEMA,
|
||||
E46GRectifiedDetectorBakeoffError,
|
||||
build_e46g_rectified_detector_bakeoff,
|
||||
read_e46g_rectified_detector_bakeoff,
|
||||
)
|
||||
|
||||
|
||||
def test_e46g_freezes_same_calibrated_views_for_both_stock_detectors(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source, profile = _source_and_profile(tmp_path)
|
||||
raw = _raw_output(tmp_path, profile)
|
||||
|
||||
result = build_e46g_rectified_detector_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
|
||||
report = result["report"]
|
||||
assert report["status"] == "completed-awaiting-visual-semantic-adjudication"
|
||||
assert report["acceptance"]["official_nvidia_dewarper_executed"] is True
|
||||
assert report["acceptance"]["same_views_and_frames_for_both_candidates"] is True
|
||||
assert report["decision"]["automatic_winner_selected"] is False
|
||||
assert report["decision"]["custom_detector_or_tracker_logic_used"] is False
|
||||
assert {component["kind"] for component in report["method"]["components"]} <= {
|
||||
"source",
|
||||
"tool",
|
||||
"model",
|
||||
"algorithm",
|
||||
"runtime",
|
||||
}
|
||||
assert set(report["metrics"]) == {"trafficcamnet", "dashcamnet"}
|
||||
assert report["metrics"]["trafficcamnet"]["view_frame_count"] == 6
|
||||
assert report["metrics"]["trafficcamnet"]["track_observation_count"] == 3
|
||||
assert report["metrics"]["dashcamnet"]["track_observation_count"] == 3
|
||||
assert result["frames"]["trafficcamnet-front"][0]["source_frame_index"] == 0
|
||||
assert (
|
||||
result["frames"]["dashcamnet-right"][0]["objects"][0]["object_id"]
|
||||
== "dashcamnet-right-nvdcf-7"
|
||||
)
|
||||
assert result["manifest"]["authority"]["candidate_accepted"] is False
|
||||
|
||||
repeated = build_e46g_rectified_detector_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
assert repeated["result_id"] == result["result_id"]
|
||||
|
||||
|
||||
def test_e46g_rejects_tampered_comparison_video(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path)
|
||||
raw = _raw_output(tmp_path, profile)
|
||||
result = build_e46g_rectified_detector_bakeoff(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
result["comparison_paths"]["trafficcamnet"].write_bytes(b"changed")
|
||||
with pytest.raises(E46GRectifiedDetectorBakeoffError, match="artifact changed"):
|
||||
read_e46g_rectified_detector_bakeoff(result["result_root"])
|
||||
|
||||
|
||||
def _source_and_profile(tmp_path: Path) -> tuple[Path, Path]:
|
||||
repository_profile = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46g_rectified_detector_bakeoff_profile.json"
|
||||
)
|
||||
profile_value = json.loads(repository_profile.read_text(encoding="utf-8"))
|
||||
source = tmp_path / "source-job"
|
||||
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
|
||||
camera.mkdir(parents=True)
|
||||
rows = [
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording-index/v1",
|
||||
"kind": "media",
|
||||
"sequence": sequence,
|
||||
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
|
||||
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
|
||||
}
|
||||
for sequence in range(1, 4490)
|
||||
]
|
||||
index_path = camera / "index.jsonl"
|
||||
index_path.write_text(
|
||||
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
summary_path = camera / "summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording/v1",
|
||||
"stream_sha256": "1" * 64,
|
||||
"segment_count": 4489,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
profile_value["source"].update(
|
||||
{
|
||||
"job_id": "recorded-test",
|
||||
"session_id": "test-session",
|
||||
"stream_sha256": "1" * 64,
|
||||
"archive_index_sha256": _sha(index_path),
|
||||
"archive_summary_sha256": _sha(summary_path),
|
||||
}
|
||||
)
|
||||
profile_value["selection"].update(
|
||||
{
|
||||
"first_source_frame_index": 0,
|
||||
"last_source_frame_index": 1,
|
||||
"frame_count": 2,
|
||||
}
|
||||
)
|
||||
(source / "job.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.compute-job/v1",
|
||||
"job_id": "recorded-test",
|
||||
"input": {
|
||||
"session_id": "test-session",
|
||||
"source_id": "sensor.camera.right",
|
||||
"segment_count": 4489,
|
||||
"archive_index_sha256": _sha(index_path),
|
||||
"archive_summary_sha256": _sha(summary_path),
|
||||
"timeline": {"start_seconds": 10.0, "end_seconds": 459.0},
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
profile = tmp_path / "profile.json"
|
||||
profile.write_text(json.dumps(profile_value, indent=2) + "\n", encoding="utf-8")
|
||||
return source, profile
|
||||
|
||||
|
||||
def _raw_output(tmp_path: Path, profile_path: Path) -> Path:
|
||||
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||
raw = tmp_path / "raw"
|
||||
geometry_runtime: dict[str, object] = {}
|
||||
for view in ("left", "front", "right"):
|
||||
full = _file(raw / "geometry" / f"{view}.mp4", f"full-{view}")
|
||||
sample = _file(raw / "samples" / f"{view}.mp4", f"sample-{view}")
|
||||
geometry_runtime[view] = {
|
||||
"dewarper_config_sha256": profile["rectification"]["views"][view]["config_sha256"],
|
||||
"dewarper_log_sha256": "2" * 64,
|
||||
"full_rectified_video_path": f"geometry/{view}.mp4",
|
||||
"full_rectified_video_sha256": _sha(full),
|
||||
"sample_video_path": f"samples/{view}.mp4",
|
||||
"sample_video_sha256": _sha(sample),
|
||||
"full_frame_count": profile["rectification"]["expected_full_frame_count"],
|
||||
"retained_source_frame_index_range": profile["rectification"][
|
||||
"retained_source_frame_index_range"
|
||||
],
|
||||
"excluded_source_tail_frame_count": profile["rectification"][
|
||||
"excluded_source_tail_frame_count"
|
||||
],
|
||||
"sample_frame_count": 2,
|
||||
}
|
||||
candidates_runtime: dict[str, object] = {}
|
||||
for candidate in ("trafficcamnet", "dashcamnet"):
|
||||
runs: dict[str, object] = {}
|
||||
for view in ("left", "front", "right"):
|
||||
root = raw / "runs" / candidate / view
|
||||
for directory in ("detections", "tracks"):
|
||||
(root / directory).mkdir(parents=True, exist_ok=True)
|
||||
for frame in range(2):
|
||||
target = root / directory / f"00_000_{frame:06d}.txt"
|
||||
target.write_text(
|
||||
_kitti_row(directory == "tracks") if frame == 0 else "",
|
||||
encoding="utf-8",
|
||||
)
|
||||
overlay = _file(root / "overlay.mp4", f"overlay-{candidate}-{view}")
|
||||
log = _file(root / "deepstream.log", "success")
|
||||
runs[view] = {
|
||||
"overlay_path": f"runs/{candidate}/{view}/overlay.mp4",
|
||||
"overlay_sha256": _sha(overlay),
|
||||
"deepstream_log_path": f"runs/{candidate}/{view}/deepstream.log",
|
||||
"deepstream_log_sha256": _sha(log),
|
||||
"tracker_config_sha256": "3" * 64,
|
||||
"model_engine_sha256": "4" * 64,
|
||||
"frame_count": 2,
|
||||
"deepstream_exit_code": 0,
|
||||
}
|
||||
candidates_runtime[candidate] = {
|
||||
"model_sha256": profile["candidates"][candidate]["model_sha256"],
|
||||
"deepstream_app_config_sha256": profile["candidates"][candidate][
|
||||
"deepstream_app_config_sha256"
|
||||
],
|
||||
"detector_config_sha256": profile["candidates"][candidate]["detector_config_sha256"],
|
||||
"parser_library_sha256": (
|
||||
profile["trafficcamnet_parser"]["library_sha256"]
|
||||
if candidate == "trafficcamnet"
|
||||
else None
|
||||
),
|
||||
"runs": runs,
|
||||
}
|
||||
comparison_runtime: dict[str, object] = {}
|
||||
for candidate in ("trafficcamnet", "dashcamnet"):
|
||||
video = _file(raw / "comparison" / f"{candidate}.mp4", candidate)
|
||||
comparison_runtime[candidate] = {
|
||||
"video_path": f"comparison/{candidate}.mp4",
|
||||
"video_sha256": _sha(video),
|
||||
"frame_count": 2,
|
||||
"view_order": ["left", "front", "right"],
|
||||
}
|
||||
_file(raw / "worker.log", "completed")
|
||||
image = profile["runtime"]["container_image"]
|
||||
runtime = {
|
||||
"schema_version": E46G_RUNTIME_SCHEMA,
|
||||
"status": "completed",
|
||||
"worker_host": "TEST-WORKER-006",
|
||||
"gpu_name": "Synthetic RTX",
|
||||
"container_image": image,
|
||||
"container_image_digest": image.rsplit("@sha256:", 1)[1],
|
||||
"source_stream_sha256": profile["source"]["stream_sha256"],
|
||||
"first_source_frame_index": 0,
|
||||
"sample_frame_count": 2,
|
||||
"geometry": geometry_runtime,
|
||||
"candidates": candidates_runtime,
|
||||
"comparison": comparison_runtime,
|
||||
}
|
||||
(raw / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n", encoding="utf-8")
|
||||
return raw
|
||||
|
||||
|
||||
def _file(path: Path, value: str) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(value.encode())
|
||||
return path
|
||||
|
||||
|
||||
def _kitti_row(tracked: bool) -> str:
|
||||
identity = " 7" if tracked else ""
|
||||
return f"car{identity} 0.0 0 0.0 10 20 300 400 0 0 0 0 0 0 0 0.91\n"
|
||||
|
||||
|
||||
def _sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _module() -> object:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "prepare_e46g_worker_package.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("e46g_worker_package_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_e46g_package_is_deterministic_calibrated_and_stock(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
kwargs = {
|
||||
"repository_root": repository,
|
||||
"profile_path": repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46g_rectified_detector_bakeoff_profile.json",
|
||||
"parser_library_path": repository
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e46e"
|
||||
/ "parser"
|
||||
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
|
||||
/ "libnvds_infercustomparser_tao.so",
|
||||
"output_root": tmp_path,
|
||||
}
|
||||
package = module.build_e46g_worker_package(**kwargs)
|
||||
repeated = module.build_e46g_worker_package(**kwargs)
|
||||
manifest = module.validate_e46g_worker_package(package)
|
||||
assert repeated == package
|
||||
assert package.name == f"e46g-worker-package-{manifest['identity_sha256']}"
|
||||
identity = manifest["identity"]
|
||||
assert identity["classification"] == (
|
||||
"minimal-factory-kb4-stock-nvidia-detector-bakeoff-package"
|
||||
)
|
||||
assert identity["calibration"]["calibration_sha256"] == (
|
||||
"05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9"
|
||||
)
|
||||
assert identity["rectification"]["provider"] == "NVIDIA Gst-nvdewarper"
|
||||
assert identity["rectification"]["expected_full_frame_count"] == 4488
|
||||
assert identity["rectification"]["retained_source_frame_index_range"] == [0, 4487]
|
||||
assert identity["selection"]["frame_count"] == 600
|
||||
assert all(
|
||||
candidate["custom_postprocessing"] is False for candidate in identity["candidates"].values()
|
||||
)
|
||||
assert identity["tracker"]["custom_association"] is False
|
||||
assert identity["tracker"]["custom_hold_or_stitch"] is False
|
||||
assert len(manifest["artifacts"]) == 19
|
||||
|
||||
|
||||
def test_e46g_package_rejects_unexpected_member(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
package = module.build_e46g_worker_package(
|
||||
repository_root=repository,
|
||||
profile_path=repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46g_rectified_detector_bakeoff_profile.json",
|
||||
parser_library_path=repository
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e46e"
|
||||
/ "parser"
|
||||
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
|
||||
/ "libnvds_infercustomparser_tao.so",
|
||||
output_root=tmp_path,
|
||||
)
|
||||
(package / "unexpected.txt").write_text("not admitted\n", encoding="utf-8")
|
||||
with pytest.raises(module.E46GWorkerPackageError, match="file set changed"):
|
||||
module.validate_e46g_worker_package(package)
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.e46h_full_rectified_front_replay import (
|
||||
E46H_RUNTIME_SCHEMA,
|
||||
E46HFullRectifiedFrontReplayError,
|
||||
build_e46h_full_rectified_front_replay,
|
||||
read_e46h_full_rectified_front_replay,
|
||||
)
|
||||
|
||||
|
||||
def test_e46h_freezes_full_retained_front_route(tmp_path: Path) -> None:
|
||||
source, profile = _source_and_profile(tmp_path)
|
||||
raw = _raw_output(tmp_path, profile)
|
||||
result = build_e46h_full_rectified_front_replay(
|
||||
source_job_root=source,
|
||||
raw_root=raw,
|
||||
profile_path=profile,
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
|
||||
report = result["report"]
|
||||
assert report["status"] == "completed-awaiting-full-route-visual-review"
|
||||
assert report["metrics"]["frame_count"] == 4488
|
||||
assert report["acceptance"]["retained_route_accounted"] is True
|
||||
assert report["acceptance"]["terminal_source_frame_excluded"] is True
|
||||
assert report["acceptance"]["full_visual_review_completed"] is False
|
||||
assert report["decision"]["selected_provider"] == "front-trafficcamnet-stock-nvdcf"
|
||||
assert report["decision"]["custom_detector_or_tracker_logic_used"] is False
|
||||
assert result["frames"][0]["source_frame_index"] == 0
|
||||
assert result["frames"][-1]["source_frame_index"] == 4487
|
||||
assert result["manifest"]["authority"]["candidate_accepted"] is False
|
||||
|
||||
result["overlay_path"].write_bytes(b"changed")
|
||||
with pytest.raises(E46HFullRectifiedFrontReplayError, match="artifact changed"):
|
||||
read_e46h_full_rectified_front_replay(result["result_root"])
|
||||
|
||||
|
||||
def _source_and_profile(tmp_path: Path) -> tuple[Path, Path]:
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
profile_value = json.loads(
|
||||
(
|
||||
repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46h_full_rectified_front_replay_profile.json"
|
||||
).read_text(encoding="utf-8")
|
||||
)
|
||||
source = tmp_path / "source-job"
|
||||
camera = source / "input" / "camera" / "sensor.camera.right" / "epoch-1"
|
||||
camera.mkdir(parents=True)
|
||||
rows = [
|
||||
{
|
||||
"schema_version": "missioncore.camera-recording-index/v1",
|
||||
"kind": "media",
|
||||
"sequence": sequence,
|
||||
"session_monotonic_ns": 1_000_000_000 + (sequence - 1) * 100_000_000,
|
||||
"sha256": hashlib.sha256(f"frame-{sequence}".encode()).hexdigest(),
|
||||
}
|
||||
for sequence in range(1, 4490)
|
||||
]
|
||||
index_path = camera / "index.jsonl"
|
||||
index_path.write_text(
|
||||
"".join(json.dumps(row, sort_keys=True) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
summary_path = camera / "summary.json"
|
||||
summary_path.write_text(
|
||||
json.dumps({"stream_sha256": "1" * 64, "segment_count": 4489}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
profile_value["source"].update(
|
||||
{
|
||||
"job_id": "recorded-test",
|
||||
"session_id": "test-session",
|
||||
"stream_sha256": "1" * 64,
|
||||
"archive_index_sha256": _sha(index_path),
|
||||
"archive_summary_sha256": _sha(summary_path),
|
||||
}
|
||||
)
|
||||
(source / "job.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "missioncore.compute-job/v1",
|
||||
"job_id": "recorded-test",
|
||||
"input": {
|
||||
"session_id": "test-session",
|
||||
"source_id": "sensor.camera.right",
|
||||
"segment_count": 4489,
|
||||
"archive_index_sha256": _sha(index_path),
|
||||
"archive_summary_sha256": _sha(summary_path),
|
||||
"timeline": {"start_seconds": 10.0, "end_seconds": 459.0},
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
profile = tmp_path / "profile.json"
|
||||
profile.write_text(json.dumps(profile_value) + "\n", encoding="utf-8")
|
||||
return source, profile
|
||||
|
||||
|
||||
def _raw_output(tmp_path: Path, profile_path: Path) -> Path:
|
||||
profile = json.loads(profile_path.read_text(encoding="utf-8"))
|
||||
raw = tmp_path / "raw"
|
||||
front = _file(raw / "geometry" / "front.mp4", "front")
|
||||
front_log = _file(raw / "geometry" / "front.log", "dewarper")
|
||||
overlay = _file(raw / "run" / "overlay.mp4", "overlay")
|
||||
deepstream_log = _file(raw / "run" / "deepstream.log", "deepstream")
|
||||
_file(raw / "worker.log", "completed")
|
||||
for directory in (raw / "run" / "detections", raw / "run" / "tracks"):
|
||||
directory.mkdir(parents=True)
|
||||
for frame in range(4488):
|
||||
(directory / f"00_000_{frame:06d}.txt").touch()
|
||||
image = profile["runtime"]["container_image"]
|
||||
runtime = {
|
||||
"schema_version": E46H_RUNTIME_SCHEMA,
|
||||
"status": "completed",
|
||||
"worker_host": "TEST-WORKER-006",
|
||||
"gpu_name": "Synthetic RTX",
|
||||
"container_image": image,
|
||||
"container_image_digest": image.rsplit("@sha256:", 1)[1],
|
||||
"source_stream_sha256": profile["source"]["stream_sha256"],
|
||||
"frame_count": 4488,
|
||||
"retained_source_frame_index_range": [0, 4487],
|
||||
"geometry": {
|
||||
"video_path": "geometry/front.mp4",
|
||||
"video_sha256": _sha(front),
|
||||
"log_path": "geometry/front.log",
|
||||
"log_sha256": _sha(front_log),
|
||||
"config_sha256": profile["rectification"]["config_sha256"],
|
||||
"frame_count": 4488,
|
||||
},
|
||||
"run": {
|
||||
"overlay_path": "run/overlay.mp4",
|
||||
"overlay_sha256": _sha(overlay),
|
||||
"deepstream_log_path": "run/deepstream.log",
|
||||
"deepstream_log_sha256": _sha(deepstream_log),
|
||||
"model_sha256": profile["detector"]["model_sha256"],
|
||||
"model_engine_sha256": "2" * 64,
|
||||
"parser_library_sha256": profile["parser"]["library_sha256"],
|
||||
"deepstream_app_config_sha256": profile["detector"][
|
||||
"deepstream_app_config_sha256"
|
||||
],
|
||||
"detector_config_sha256": profile["detector"]["detector_config_sha256"],
|
||||
"tracker_config_sha256": "3" * 64,
|
||||
"frame_count": 4488,
|
||||
},
|
||||
}
|
||||
(raw / "runtime.json").write_text(json.dumps(runtime) + "\n", encoding="utf-8")
|
||||
return raw
|
||||
|
||||
|
||||
def _file(path: Path, value: str) -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(value, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def _sha(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _module() -> object:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "prepare_e46h_worker_package.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("e46h_worker_package_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_e46h_package_is_deterministic_front_only_and_stock(tmp_path: Path) -> None:
|
||||
module = _module()
|
||||
repository = Path(__file__).resolve().parents[1]
|
||||
kwargs = {
|
||||
"repository_root": repository,
|
||||
"profile_path": repository
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "e46h_full_rectified_front_replay_profile.json",
|
||||
"parser_library_path": repository
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e46e"
|
||||
/ "parser"
|
||||
/ "a18d85dae674a088549c5f9b8fda53c640f4fcbd88a41f4c2cb1f4e3ea8878ee"
|
||||
/ "libnvds_infercustomparser_tao.so",
|
||||
"output_root": tmp_path,
|
||||
}
|
||||
package = module.build_e46h_worker_package(**kwargs)
|
||||
repeated = module.build_e46h_worker_package(**kwargs)
|
||||
manifest = module.validate_e46h_worker_package(package)
|
||||
assert repeated == package
|
||||
assert package.name == f"e46h-worker-package-{manifest['identity_sha256']}"
|
||||
identity = manifest["identity"]
|
||||
assert identity["classification"] == "minimal-full-front-stock-nvidia-replay-package"
|
||||
assert identity["rectification"]["view"] == "front"
|
||||
assert identity["selection"]["frame_count"] == 4488
|
||||
assert identity["selection"]["last_source_frame_index"] == 4487
|
||||
assert identity["detector"]["custom_postprocessing"] is False
|
||||
assert identity["tracker"]["custom_association"] is False
|
||||
assert identity["tracker"]["custom_hold_or_stitch"] is False
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.l34_right_yolox_truth_island_freeze import (
|
||||
L34_PREDICTION_SCHEMA,
|
||||
L34RightYoloxTruthIslandError,
|
||||
freeze_l34_candidate_predictions,
|
||||
)
|
||||
|
||||
|
||||
def _reference(frame_index: int, sequence: int) -> dict[str, object]:
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence + 10,
|
||||
"frame_index": frame_index,
|
||||
"group_id": "temporal-a",
|
||||
"session_seconds": 42.5,
|
||||
"sha256": "a" * 64,
|
||||
}
|
||||
|
||||
|
||||
def _frame(frame_index: int) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.rectified-yolox-frame/v1",
|
||||
"frame_index": frame_index,
|
||||
"detections": [
|
||||
{
|
||||
"label": "bus",
|
||||
"score": 0.91,
|
||||
"bbox_xyxy": [10.0, 20.0, 30.0, 40.0],
|
||||
},
|
||||
{
|
||||
"label": "person",
|
||||
"score": 0.72,
|
||||
"bbox_xyxy": [1.0, 2.0, 3.0, 4.0],
|
||||
},
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.24,
|
||||
"bbox_xyxy": [5.0, 6.0, 7.0, 8.0],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_freezes_only_admitted_right_camera_candidate_classes() -> None:
|
||||
predictions = freeze_l34_candidate_predictions(
|
||||
references=(_reference(70, 1),),
|
||||
detector_frames={70: _frame(70)},
|
||||
minimum_score=0.25,
|
||||
)
|
||||
|
||||
assert predictions == (
|
||||
{
|
||||
"schema_version": L34_PREDICTION_SCHEMA,
|
||||
"candidate_id": "yolox-s-kb4-core3",
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 11,
|
||||
"frame_index": 70,
|
||||
"session_seconds": 42.5,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"group_id": "temporal-a",
|
||||
"predictions": [
|
||||
{
|
||||
"label": "heavy_vehicle",
|
||||
"score": 0.91,
|
||||
"bbox_xyxy": [10.0, 20.0, 30.0, 40.0],
|
||||
},
|
||||
{
|
||||
"label": "person",
|
||||
"score": 0.72,
|
||||
"bbox_xyxy": [1.0, 2.0, 3.0, 4.0],
|
||||
},
|
||||
],
|
||||
"truth_joined": False,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_freeze_fails_closed_on_incomplete_frame_coverage() -> None:
|
||||
with pytest.raises(
|
||||
L34RightYoloxTruthIslandError,
|
||||
match="coverage is incomplete",
|
||||
):
|
||||
freeze_l34_candidate_predictions(
|
||||
references=(_reference(70, 1), _reference(71, 2)),
|
||||
detector_frames={70: _frame(70)},
|
||||
minimum_score=0.25,
|
||||
)
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from k1link.compute.l34a_assisted_yolox_error_audit import (
|
||||
_aggregate,
|
||||
_audit_case,
|
||||
)
|
||||
|
||||
|
||||
def _prediction_row() -> dict[str, object]:
|
||||
return {
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 2,
|
||||
"frame_index": 70,
|
||||
"group_id": "anchor-a",
|
||||
"session_seconds": 4.2,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"predictions": [
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.9,
|
||||
"bbox_xyxy": [100.0, 100.0, 300.0, 300.0],
|
||||
},
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.7,
|
||||
"bbox_xyxy": [120.0, 120.0, 280.0, 280.0],
|
||||
},
|
||||
{
|
||||
"label": "motorcycle",
|
||||
"score": 0.6,
|
||||
"bbox_xyxy": [400.0, 200.0, 520.0, 420.0],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _annotation_frame() -> dict[str, object]:
|
||||
return {
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 2,
|
||||
"frame_index": 70,
|
||||
"source_sha256": "a" * 64,
|
||||
"objects": [
|
||||
{
|
||||
"object_id": "car-1",
|
||||
"category": "car",
|
||||
"proposed_label": None,
|
||||
"origin": "frozen_candidate_seed",
|
||||
"box_xyxy": [100.0, 100.0, 300.0, 300.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
},
|
||||
{
|
||||
"object_id": "stroller-1",
|
||||
"category": "unmapped",
|
||||
"proposed_label": "Детская коляска",
|
||||
"origin": "manual",
|
||||
"box_xyxy": [400.0, 200.0, 520.0, 420.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
},
|
||||
{
|
||||
"object_id": "person-1",
|
||||
"category": "person",
|
||||
"proposed_label": None,
|
||||
"origin": "manual",
|
||||
"box_xyxy": [10.0, 10.0, 60.0, 160.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_assisted_audit_distinguishes_duplicate_mismatch_and_miss() -> None:
|
||||
case = _audit_case(
|
||||
prediction_row=_prediction_row(),
|
||||
annotation_frame=_annotation_frame(),
|
||||
)
|
||||
|
||||
assert case["summary"] == {
|
||||
"prediction_count": 3,
|
||||
"reference_count": 3,
|
||||
"true_positive": 1,
|
||||
"false_positive": 2,
|
||||
"false_negative": 2,
|
||||
"class_mismatch": 1,
|
||||
"duplicate_false_positive": 1,
|
||||
"unmatched_false_positive": 0,
|
||||
"unmatched_false_negative": 1,
|
||||
"severity_score": 7,
|
||||
}
|
||||
assert [item["verdict"] for item in case["predictions"]] == [
|
||||
"true_positive",
|
||||
"duplicate_false_positive",
|
||||
"class_mismatch",
|
||||
]
|
||||
assert case["annotations"][1]["display_category"] == (
|
||||
"unmapped:Детская коляска"
|
||||
)
|
||||
|
||||
|
||||
def test_assisted_alignment_metrics_remain_descriptive() -> None:
|
||||
case = _audit_case(
|
||||
prediction_row=_prediction_row(),
|
||||
annotation_frame=_annotation_frame(),
|
||||
)
|
||||
|
||||
metrics = _aggregate((case,))
|
||||
|
||||
assert metrics["precision_iou50"] == 1 / 3
|
||||
assert metrics["recall_iou50"] == 1 / 3
|
||||
assert metrics["f1_iou50"] == 1 / 3
|
||||
assert metrics["custom_reference_count"] == 1
|
||||
assert metrics["error_case_count"] == 1
|
||||
@@ -0,0 +1,100 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from k1link.compute.l34a_assisted_yolox_error_audit import _audit_case
|
||||
from k1link.compute.l34b_nested_box_consolidation_shadow import (
|
||||
consolidate_l34b_prediction_row,
|
||||
evaluate_l34b_shadow,
|
||||
)
|
||||
|
||||
|
||||
def _row(sequence: int = 1) -> dict[str, object]:
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence + 10,
|
||||
"group_id": "nested-box",
|
||||
"session_seconds": float(sequence),
|
||||
"source_image_sha256": f"{sequence % 10}" * 64,
|
||||
"predictions": [
|
||||
{
|
||||
"label": "heavy_vehicle",
|
||||
"score": 0.71,
|
||||
"bbox_xyxy": [100.0, 100.0, 140.0, 190.0],
|
||||
},
|
||||
{
|
||||
"label": "heavy_vehicle",
|
||||
"score": 0.69,
|
||||
"bbox_xyxy": [90.0, 99.0, 141.0, 191.0],
|
||||
},
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.8,
|
||||
"bbox_xyxy": [300.0, 100.0, 360.0, 180.0],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _annotation(sequence: int = 1) -> dict[str, object]:
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence + 10,
|
||||
"source_sha256": f"{sequence % 10}" * 64,
|
||||
"objects": [
|
||||
{
|
||||
"object_id": f"truck-{sequence}",
|
||||
"category": "heavy_vehicle",
|
||||
"proposed_label": None,
|
||||
"origin": "manual",
|
||||
"box_xyxy": [90.0, 99.0, 141.0, 191.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
},
|
||||
{
|
||||
"object_id": f"car-{sequence}",
|
||||
"category": "car",
|
||||
"proposed_label": None,
|
||||
"origin": "manual",
|
||||
"box_xyxy": [300.0, 100.0, 360.0, 180.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_consolidates_only_same_category_nested_boxes() -> None:
|
||||
projected, consolidations = consolidate_l34b_prediction_row(_row())
|
||||
|
||||
assert len(projected["predictions"]) == 2
|
||||
assert consolidations[0]["source_prediction_indices"] == [1, 2]
|
||||
assert projected["predictions"][0] == {
|
||||
"label": "heavy_vehicle",
|
||||
"score": 0.71,
|
||||
"bbox_xyxy": [90.0, 99.0, 141.0, 191.0],
|
||||
"source_prediction_indices": [1, 2],
|
||||
}
|
||||
assert projected["predictions"][1]["source_prediction_indices"] == [3]
|
||||
|
||||
|
||||
def test_shadow_reports_regression_free_false_positive_reduction() -> None:
|
||||
rows = tuple(_row(sequence) for sequence in range(1, 33))
|
||||
annotations = tuple(_annotation(sequence) for sequence in range(1, 33))
|
||||
before = tuple(
|
||||
_audit_case(prediction_row=row, annotation_frame=annotation)
|
||||
for row, annotation in zip(rows, annotations, strict=True)
|
||||
)
|
||||
|
||||
cases, metrics = evaluate_l34b_shadow(
|
||||
prediction_rows=rows,
|
||||
annotation_frames=annotations,
|
||||
before_cases=before,
|
||||
)
|
||||
|
||||
assert len(cases) == 32
|
||||
assert metrics["consolidation_count"] == 32
|
||||
assert metrics["delta"]["true_positive"] == 0
|
||||
assert metrics["delta"]["false_positive"] == -32
|
||||
assert metrics["delta"]["false_negative"] == 0
|
||||
assert metrics["assisted_regression_free"] is True
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.l34c_tile_seam_stitch_shadow import (
|
||||
L34C_PROVENANCE_SCHEMA,
|
||||
L34CTileSeamStitchError,
|
||||
apply_l34c_stitches,
|
||||
bind_l34c_prediction_provenance,
|
||||
find_l34c_temporal_stitches,
|
||||
)
|
||||
|
||||
|
||||
def _frozen_row(*, sequence: int = 1, frame_index: int = 100) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.l34-right-yolox-truth-island-prediction/v1",
|
||||
"candidate_id": "yolox-s-kb4-core3",
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": frame_index,
|
||||
"session_seconds": frame_index / 10.0,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"group_id": "clip-close-car",
|
||||
"predictions": [
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.95,
|
||||
"bbox_xyxy": [230.0, 220.0, 385.0, 340.0],
|
||||
},
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.90,
|
||||
"bbox_xyxy": [182.0, 210.0, 266.0, 338.0],
|
||||
},
|
||||
],
|
||||
"truth_joined": False,
|
||||
}
|
||||
|
||||
|
||||
def _detector_frame(frame_index: int = 100) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.rectified-yolox-frame/v1",
|
||||
"frame_index": frame_index,
|
||||
"detections": [
|
||||
{
|
||||
"label": "car",
|
||||
"class_id": 2,
|
||||
"score": 0.95,
|
||||
"bbox_xyxy": [230.0, 220.0, 385.0, 340.0],
|
||||
"raw_center_xy": [290.0, 280.0],
|
||||
"rectification_tile": "front",
|
||||
"valid_fov_fraction": 1.0,
|
||||
},
|
||||
{
|
||||
"label": "car",
|
||||
"class_id": 2,
|
||||
"score": 0.90,
|
||||
"bbox_xyxy": [182.0, 210.0, 266.0, 338.0],
|
||||
"raw_center_xy": [230.0, 275.0],
|
||||
"rectification_tile": "left",
|
||||
"valid_fov_fraction": 1.0,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _provenance_run(length: int) -> tuple[dict[str, object], ...]:
|
||||
frozen = tuple(
|
||||
_frozen_row(sequence=index + 1, frame_index=100 + index)
|
||||
for index in range(length)
|
||||
)
|
||||
detector = {
|
||||
100 + index: _detector_frame(100 + index)
|
||||
for index in range(length)
|
||||
}
|
||||
return bind_l34c_prediction_provenance(
|
||||
prediction_rows=frozen,
|
||||
detector_frames=detector,
|
||||
)
|
||||
|
||||
|
||||
def test_exact_join_preserves_tile_and_raw_detector_identity() -> None:
|
||||
rows = _provenance_run(1)
|
||||
|
||||
assert rows[0]["schema_version"] == L34C_PROVENANCE_SCHEMA
|
||||
assert rows[0]["provenance_join"] == "exact-label-score-bbox"
|
||||
assert rows[0]["predictions"][0]["rectification_tile"] == "front"
|
||||
assert rows[0]["predictions"][1]["rectification_tile"] == "left"
|
||||
assert rows[0]["predictions"][0]["class_id"] == 2
|
||||
assert rows[0]["predictions"][0]["raw_label"] == "car"
|
||||
|
||||
|
||||
def test_exact_join_rejects_missing_or_ambiguous_provenance() -> None:
|
||||
frame = _detector_frame()
|
||||
frame["detections"].append(copy.deepcopy(frame["detections"][0]))
|
||||
|
||||
with pytest.raises(L34CTileSeamStitchError, match="one exact"):
|
||||
bind_l34c_prediction_provenance(
|
||||
prediction_rows=(_frozen_row(),),
|
||||
detector_frames={100: frame},
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_gate_admits_three_consecutive_frames_but_not_two() -> None:
|
||||
admitted, static_count = find_l34c_temporal_stitches(_provenance_run(3))
|
||||
rejected, rejected_static_count = find_l34c_temporal_stitches(
|
||||
_provenance_run(2)
|
||||
)
|
||||
|
||||
assert static_count == 3
|
||||
assert set(admitted) == {1, 2, 3}
|
||||
assert all(items[0]["temporal_run_length"] == 3 for items in admitted.values())
|
||||
assert rejected_static_count == 2
|
||||
assert rejected == {}
|
||||
|
||||
|
||||
def test_temporal_gate_rejects_same_tile_and_small_pairs() -> None:
|
||||
rows = list(_provenance_run(3))
|
||||
for row in rows:
|
||||
row["predictions"][1]["rectification_tile"] = "front"
|
||||
same_tile, same_tile_count = find_l34c_temporal_stitches(tuple(rows))
|
||||
|
||||
small_rows = list(_provenance_run(3))
|
||||
for row in small_rows:
|
||||
row["predictions"][0]["bbox_xyxy"] = [230.0, 220.0, 260.0, 250.0]
|
||||
row["predictions"][1]["bbox_xyxy"] = [220.0, 218.0, 245.0, 252.0]
|
||||
small, small_count = find_l34c_temporal_stitches(tuple(small_rows))
|
||||
|
||||
assert same_tile_count == 0
|
||||
assert same_tile == {}
|
||||
assert small_count == 0
|
||||
assert small == {}
|
||||
|
||||
|
||||
def test_apply_stitch_unions_geometry_and_preserves_source_tiles() -> None:
|
||||
row = _provenance_run(3)[0]
|
||||
admitted, _ = find_l34c_temporal_stitches(_provenance_run(3))
|
||||
|
||||
projected = apply_l34c_stitches(row, admitted[1])
|
||||
|
||||
assert projected["predictions"] == [
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.95,
|
||||
"bbox_xyxy": [182.0, 210.0, 385.0, 340.0],
|
||||
"source_prediction_indices": [1, 2],
|
||||
"source_rectification_tiles": ["front", "left"],
|
||||
"temporal_run_id": "clip-close-car:car:100-102",
|
||||
"temporal_run_length": 3,
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,124 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.l34c_tile_seam_stitch_shadow import (
|
||||
bind_l34c_prediction_provenance,
|
||||
)
|
||||
from k1link.compute.l34d_cumulative_postprocessing_candidate import (
|
||||
L34DCumulativeCandidateError,
|
||||
compose_l34d_prediction_row,
|
||||
)
|
||||
|
||||
|
||||
def _row() -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": "missioncore.l34-right-yolox-truth-island-prediction/v1",
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 1,
|
||||
"frame_index": 100,
|
||||
"session_seconds": 1.0,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"group_id": "composition",
|
||||
"predictions": [
|
||||
{"label": "heavy_vehicle", "score": 0.71, "bbox_xyxy": [10.0, 10.0, 30.0, 40.0]},
|
||||
{"label": "heavy_vehicle", "score": 0.69, "bbox_xyxy": [9.0, 9.0, 31.0, 41.0]},
|
||||
{"label": "car", "score": 0.95, "bbox_xyxy": [230.0, 220.0, 385.0, 340.0]},
|
||||
{"label": "car", "score": 0.90, "bbox_xyxy": [182.0, 210.0, 266.0, 338.0]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _detector_frame() -> dict[str, object]:
|
||||
detections = []
|
||||
for index, prediction in enumerate(_row()["predictions"]):
|
||||
detections.append({
|
||||
**prediction,
|
||||
"class_id": 7 if index < 2 else 2,
|
||||
"raw_center_xy": [20.0 + index, 20.0 + index],
|
||||
"rectification_tile": "front" if index in (0, 2) else "left",
|
||||
"valid_fov_fraction": 1.0,
|
||||
})
|
||||
return {
|
||||
"schema_version": "missioncore.rectified-yolox-frame/v1",
|
||||
"frame_index": 100,
|
||||
"detections": detections,
|
||||
}
|
||||
|
||||
|
||||
def _provenance() -> dict[str, object]:
|
||||
return bind_l34c_prediction_provenance(
|
||||
prediction_rows=(_row(),),
|
||||
detector_frames={100: _detector_frame()},
|
||||
)[0]
|
||||
|
||||
|
||||
def _operation(indices: list[int], operation_type: str) -> dict[str, object]:
|
||||
predictions = _row()["predictions"]
|
||||
members = [predictions[index - 1] for index in indices]
|
||||
boxes = [member["bbox_xyxy"] for member in members]
|
||||
operation = {
|
||||
"category": members[0]["label"],
|
||||
"source_prediction_indices": indices,
|
||||
"source_scores": [member["score"] for member in members],
|
||||
"source_boxes_xyxy": boxes,
|
||||
"merged_score": max(member["score"] for member in members),
|
||||
"merged_box_xyxy": [
|
||||
min(box[0] for box in boxes),
|
||||
min(box[1] for box in boxes),
|
||||
max(box[2] for box in boxes),
|
||||
max(box[3] for box in boxes),
|
||||
],
|
||||
}
|
||||
if operation_type == "temporal-tile-seam-stitch":
|
||||
operation.update({
|
||||
"source_tiles": ["front", "left"],
|
||||
"temporal_run_id": "composition:car:100-102",
|
||||
"temporal_run_length": 3,
|
||||
})
|
||||
return operation
|
||||
|
||||
|
||||
def test_composes_disjoint_nested_and_seam_operations_once() -> None:
|
||||
projected, operations = compose_l34d_prediction_row(
|
||||
_provenance(),
|
||||
consolidations=(_operation([1, 2], "nested-box-consolidation"),),
|
||||
stitches=(_operation([3, 4], "temporal-tile-seam-stitch"),),
|
||||
)
|
||||
|
||||
assert len(projected["predictions"]) == 2
|
||||
assert [
|
||||
item["source_prediction_indices"] for item in projected["predictions"]
|
||||
] == [[1, 2], [3, 4]]
|
||||
assert [item["operation_types"] for item in projected["predictions"]] == [
|
||||
["nested-box-consolidation"],
|
||||
["temporal-tile-seam-stitch"],
|
||||
]
|
||||
assert {item["operation_type"] for item in operations} == {
|
||||
"nested-box-consolidation",
|
||||
"temporal-tile-seam-stitch",
|
||||
}
|
||||
|
||||
|
||||
def test_rejects_overlapping_operation_sets() -> None:
|
||||
with pytest.raises(L34DCumulativeCandidateError, match="overlap"):
|
||||
compose_l34d_prediction_row(
|
||||
_provenance(),
|
||||
consolidations=(_operation([1, 2], "nested-box-consolidation"),),
|
||||
stitches=(_operation([1, 2], "temporal-tile-seam-stitch"),),
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_operation_payload_drift() -> None:
|
||||
operation = _operation([1, 2], "nested-box-consolidation")
|
||||
operation["source_boxes_xyxy"] = copy.deepcopy(operation["source_boxes_xyxy"])
|
||||
operation["source_boxes_xyxy"][0][0] += 1.0
|
||||
|
||||
with pytest.raises(L34DCumulativeCandidateError, match="payload"):
|
||||
compose_l34d_prediction_row(
|
||||
_provenance(),
|
||||
consolidations=(operation,),
|
||||
stitches=(),
|
||||
)
|
||||
@@ -0,0 +1,152 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from k1link.compute.l34e_self_review_diagnostic import (
|
||||
L34ESelfReviewDiagnosticError,
|
||||
evaluate_l34e_self_review_diagnostic,
|
||||
)
|
||||
|
||||
|
||||
def _candidate_case(
|
||||
sequence: int,
|
||||
*,
|
||||
category: str = "car",
|
||||
box: list[float] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
predictions = [] if box is None else [
|
||||
{
|
||||
"prediction_index": 1,
|
||||
"category": category,
|
||||
"score": 0.8,
|
||||
"box_xyxy": box,
|
||||
"source_prediction_indices": [1],
|
||||
"source_rectification_tiles": ["center"],
|
||||
"operation_types": [],
|
||||
}
|
||||
]
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence * 10,
|
||||
"group_id": f"frame-{sequence:02d}",
|
||||
"session_seconds": float(sequence),
|
||||
"source_image_sha256": f"{sequence:064x}",
|
||||
"after_predictions": predictions,
|
||||
}
|
||||
|
||||
|
||||
def _review_frame(
|
||||
sequence: int,
|
||||
*,
|
||||
category: str = "car",
|
||||
box: list[float] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
objects = [] if box is None else [
|
||||
{
|
||||
"object_id": f"manual-{sequence:02d}",
|
||||
"category": category,
|
||||
"proposed_label": None,
|
||||
"origin": "manual",
|
||||
"box_xyxy": box,
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
}
|
||||
]
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": sequence,
|
||||
"frame_index": sequence * 10,
|
||||
"source_sha256": f"{sequence:064x}",
|
||||
"reviewed": True,
|
||||
"objects": objects,
|
||||
}
|
||||
|
||||
|
||||
def _fixture() -> tuple[tuple[dict[str, Any], ...], tuple[dict[str, Any], ...]]:
|
||||
candidates = []
|
||||
reviews = []
|
||||
for sequence in range(1, 33):
|
||||
candidate_category = "car"
|
||||
review_category = "car"
|
||||
candidate_box: list[float] | None = [10.0, 10.0, 30.0, 30.0]
|
||||
review_box: list[float] | None = [10.0, 10.0, 30.0, 30.0]
|
||||
if sequence == 2:
|
||||
review_box = [8.0, 8.0, 40.0, 40.0]
|
||||
elif sequence == 3:
|
||||
review_box = [8.0, 8.0, 40.0, 40.0]
|
||||
review_category = "person"
|
||||
elif sequence == 4:
|
||||
review_category = "person"
|
||||
elif sequence == 5:
|
||||
review_box = None
|
||||
elif sequence == 6:
|
||||
candidate_box = None
|
||||
candidates.append(
|
||||
_candidate_case(
|
||||
sequence,
|
||||
category=candidate_category,
|
||||
box=candidate_box,
|
||||
)
|
||||
)
|
||||
reviews.append(
|
||||
_review_frame(
|
||||
sequence,
|
||||
category=review_category,
|
||||
box=review_box,
|
||||
)
|
||||
)
|
||||
return tuple(candidates), tuple(reviews)
|
||||
|
||||
|
||||
def test_evaluation_separates_localization_from_unmatched_objects() -> None:
|
||||
candidates, reviews = _fixture()
|
||||
|
||||
cases, metrics = evaluate_l34e_self_review_diagnostic(
|
||||
l34d_cases=candidates,
|
||||
annotation_frames=reviews,
|
||||
)
|
||||
|
||||
diagnostic = metrics["diagnostic_association"]
|
||||
assert diagnostic == {
|
||||
"prediction_count": 31,
|
||||
"reference_count": 31,
|
||||
"associated_pair_count": 30,
|
||||
"strict_alignment": 27,
|
||||
"strict_class_mismatch": 1,
|
||||
"localization_disagreement": 1,
|
||||
"class_and_localization_disagreement": 1,
|
||||
"prediction_only": 1,
|
||||
"reference_only": 1,
|
||||
"candidate_association_coverage": 30 / 31,
|
||||
"reference_association_coverage": 30 / 31,
|
||||
"error_case_count": 5,
|
||||
}
|
||||
strict = metrics["strict_iou50"]
|
||||
assert strict["true_positive"] == 27
|
||||
assert strict["false_positive"] == 4
|
||||
assert strict["false_negative"] == 4
|
||||
assert strict["class_mismatch"] == 1
|
||||
assert cases[1]["predictions"][0]["diagnostic_verdict"] == (
|
||||
"localization_disagreement"
|
||||
)
|
||||
assert cases[2]["references"][0]["diagnostic_verdict"] == (
|
||||
"class_and_localization_disagreement"
|
||||
)
|
||||
assert cases[4]["predictions"][0]["diagnostic_verdict"] == "prediction_only"
|
||||
assert cases[5]["references"][0]["diagnostic_verdict"] == "reference_only"
|
||||
|
||||
|
||||
def test_evaluation_rejects_incomplete_coverage() -> None:
|
||||
candidates, reviews = _fixture()
|
||||
|
||||
with pytest.raises(
|
||||
L34ESelfReviewDiagnosticError,
|
||||
match="exactly 32",
|
||||
):
|
||||
evaluate_l34e_self_review_diagnostic(
|
||||
l34d_cases=candidates[:-1],
|
||||
annotation_frames=reviews,
|
||||
)
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from k1link.compute.e49_detector_truth_evaluation import (
|
||||
evaluate_frozen_detector_candidates,
|
||||
)
|
||||
from k1link.compute.l34_right_yolox_truth_island_freeze import (
|
||||
L34_PREDICTION_SCHEMA,
|
||||
)
|
||||
from k1link.compute.l35_right_yolox_truth_evaluation import (
|
||||
L35RightYoloxTruthEvaluationError,
|
||||
_require_freeze_before_truth_seal,
|
||||
l34_rows_for_sealed_truth,
|
||||
)
|
||||
|
||||
|
||||
def _l34_row() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": L34_PREDICTION_SCHEMA,
|
||||
"candidate_id": "yolox-s-kb4-core3",
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 11,
|
||||
"frame_index": 70,
|
||||
"session_seconds": 42.5,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"group_id": "anchor-011",
|
||||
"predictions": [
|
||||
{
|
||||
"label": "car",
|
||||
"score": 0.9,
|
||||
"bbox_xyxy": [10.0, 10.0, 110.0, 110.0],
|
||||
}
|
||||
],
|
||||
"truth_joined": False,
|
||||
}
|
||||
|
||||
|
||||
def _truth_row() -> dict[str, Any]:
|
||||
return {
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 11,
|
||||
"frame_index": 70,
|
||||
"session_seconds": 42.5,
|
||||
"role": "anchor",
|
||||
"group_id": "anchor-011",
|
||||
"source_image_sha256": "a" * 64,
|
||||
"hard_negative": False,
|
||||
"objects": [
|
||||
{
|
||||
"object_id": "car-1",
|
||||
"category": "car",
|
||||
"box_xyxy": [10.0, 10.0, 110.0, 110.0],
|
||||
"occluded": False,
|
||||
"truncated": False,
|
||||
"notes": None,
|
||||
}
|
||||
],
|
||||
"adjudicated": True,
|
||||
}
|
||||
|
||||
|
||||
def test_l35_adapts_l34_without_changing_identity_or_boxes() -> None:
|
||||
rows = l34_rows_for_sealed_truth((_l34_row(),))
|
||||
|
||||
assert rows == (
|
||||
{
|
||||
"candidate_id": "yolox-s-kb4-core3",
|
||||
"truth_island_sequence": 1,
|
||||
"image_id": 11,
|
||||
"frame_index": 70,
|
||||
"session_seconds": 42.5,
|
||||
"source_image_sha256": "a" * 64,
|
||||
"predictions": [
|
||||
{
|
||||
"category": "car",
|
||||
"score": 0.9,
|
||||
"box_xyxy": [10.0, 10.0, 110.0, 110.0],
|
||||
}
|
||||
],
|
||||
"truth_joined": False,
|
||||
},
|
||||
)
|
||||
metrics = evaluate_frozen_detector_candidates(
|
||||
truth_rows=(_truth_row(),),
|
||||
prediction_rows=rows,
|
||||
valid_fov_mask=Image.new("L", (800, 600), color=255),
|
||||
)
|
||||
assert metrics["yolox-s-kb4-core3"]["ap50"] == 1.0
|
||||
assert metrics["yolox-s-kb4-core3"]["candidate_winner_selected"] is False
|
||||
|
||||
|
||||
def test_l35_rejects_a_freeze_created_after_truth_was_sealed() -> None:
|
||||
with pytest.raises(
|
||||
L35RightYoloxTruthEvaluationError,
|
||||
match="postdates",
|
||||
):
|
||||
_require_freeze_before_truth_seal(
|
||||
freeze_created_at_utc="2026-08-01T02:00:00Z",
|
||||
truth_sealed_at_utc="2026-08-01T01:00:00Z",
|
||||
)
|
||||
|
||||
|
||||
def test_l35_accepts_a_freeze_created_before_truth_was_sealed() -> None:
|
||||
_require_freeze_before_truth_seal(
|
||||
freeze_created_at_utc="2026-08-01T00:30:00Z",
|
||||
truth_sealed_at_utc="2026-08-01T01:00:00Z",
|
||||
)
|
||||
@@ -76,6 +76,9 @@ def test_camera_semantic_and_connected_occupied_support_agree() -> None:
|
||||
)
|
||||
assert support.document["geometry_status"] == "agree"
|
||||
assert support.document["range_m"] == pytest.approx(2.0666666667)
|
||||
assert support.document["range_estimate_m"] == pytest.approx(2.0666666667)
|
||||
assert support.document["range_estimate_available"] is True
|
||||
assert support.document["range_support_qualified"] is True
|
||||
assert support.document["unknown_is_occupied"] is True
|
||||
assert support.document["navigation_or_safety_accepted"] is False
|
||||
assert support.occupied_source_indices.tolist() == [0, 1]
|
||||
@@ -120,8 +123,18 @@ def test_camera_only_and_surface_conflict_remain_explicit() -> None:
|
||||
profile=CameraGeometryFusionProfile(),
|
||||
)
|
||||
assert camera_only.document["geometry_status"] == "single-source-camera"
|
||||
assert camera_only.document["range_m"] is None
|
||||
assert camera_only.document["range_estimate_m"] == pytest.approx(2.0)
|
||||
assert camera_only.document["range_estimate_available"] is True
|
||||
assert camera_only.document["range_support_qualified"] is False
|
||||
assert conflict.document["geometry_status"] == "conflict"
|
||||
assert conflict.document["range_m"] is None
|
||||
assert conflict.document["range_estimate_available"] is False
|
||||
assert conflict.document["range_support_qualified"] is False
|
||||
assert unknown.document["geometry_status"] == "unknown"
|
||||
assert unknown.document["range_m"] is None
|
||||
assert unknown.document["range_estimate_available"] is False
|
||||
assert unknown.document["range_support_qualified"] is False
|
||||
|
||||
|
||||
def test_unclaimed_occupied_component_is_a_separate_geometry_layer() -> None:
|
||||
|
||||
Reference in New Issue
Block a user