feat(perception): prepare blind detector review handoff
This commit is contained in:
DCCONSTRUCTIONS
@@ -15,7 +15,7 @@ Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result.
|
||||
| Evidence storage | GO (reference index only) — E50 converts the accepted E44 audit into 2,962 verified logical references over 1,029 canonical content objects, including 1,933 exact duplicate references and 212,717,913 measured duplicate bytes. Existing artifacts remain materialized and unchanged; physical reclamation and storage migration remain unauthorized. |
|
||||
| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. |
|
||||
| E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. |
|
||||
| Detector Truth Island | PREPARED — E46 freezes 32 references with no prelabels, predictions, scores or candidate identity in the review package. Two independent reviews and adjudication are still required. |
|
||||
| Detector Truth Island | PREPARED FOR HUMAN REVIEW — E46 freezes 32 references with no prelabels, predictions, scores or candidate identity. Immutable handoff `e46-review-handoff-652f7409c8a529946a26264cea006f3afdd714e19529c731e0f66d0f03fd5668` provides two separately assigned, hash-verified CVAT/COCO reviewer packages and a fail-closed E48 converter. Two real independent reviews and adjudication are still required; the handoff is not ground truth. |
|
||||
| Detector candidate comparison | FROZEN BEFORE TRUTH — E47 freezes raw-KB4 and fixed-valid-FOV-fill predictions from the same exact Mask R-CNN checkpoint. No accuracy result or winner exists before the E46 truth seal. |
|
||||
| Detector truth/evaluation executors | READY, NOT RUN — E48 fail-closed review/adjudication sealing and separate post-seal E49 scoring are implemented. Neither result exists because real independent reviews are absent. |
|
||||
| Product interface | DEFERRED — no new windows, page anatomy or design changes are part of this stabilization increment. |
|
||||
|
||||
@@ -168,6 +168,16 @@ prelabels, predictions, scores or candidate identity. It references
|
||||
14,919,621 source bytes without copying images. The package is prepared, not
|
||||
truth: two different human reviewers and adjudication remain mandatory.
|
||||
|
||||
Operational handoff
|
||||
`e46-review-handoff-652f7409c8a529946a26264cea006f3afdd714e19529c731e0f66d0f03fd5668`
|
||||
materializes those exact 32 hash-verified source frames into two separately
|
||||
assignable CVAT/COCO slots. Both slots contain an empty annotation seed and the
|
||||
same seven-class contract; neither contains predictions, prelabels, scores or
|
||||
candidate identity. A converter validates exact frame coverage, allowed
|
||||
classes, boxes and explicit occlusion/truncation fields before producing one
|
||||
strict E48 review submission. This closes the technical handoff only: no human
|
||||
review, adjudication or ground-truth seal is implied.
|
||||
|
||||
E47 immutable candidate freeze
|
||||
`e47-detector-candidate-freeze-514bcca7a8a26313cab5ffcacca053d7a0ec6fe7cbef25f15faf3a11e48ee92f`
|
||||
stores raw-KB4 and fixed-valid-FOV-fill predictions for those exact 32 frames.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert one completed blind CVAT task into an E48 review submission."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46_review_handoff import convert_cvat_coco_to_e48_review
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--handoff-root", type=Path, required=True)
|
||||
parser.add_argument("--truth-island-root", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--reviewer-slot",
|
||||
choices=("reviewer-a", "reviewer-b"),
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument("--reviewer-id", required=True)
|
||||
parser.add_argument("--cvat-export", type=Path, required=True)
|
||||
parser.add_argument("--submitted-at-utc", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
document = convert_cvat_coco_to_e48_review(
|
||||
handoff_root=args.handoff_root,
|
||||
truth_island_root=args.truth_island_root,
|
||||
reviewer_slot=args.reviewer_slot,
|
||||
reviewer_id=args.reviewer_id,
|
||||
cvat_export_path=args.cvat_export,
|
||||
submitted_at_utc=args.submitted_at_utc,
|
||||
output_path=args.output,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"output": str(args.output.expanduser().absolute()),
|
||||
"reviewer_id": document["reviewer_id"],
|
||||
"frame_count": len(document["images"]),
|
||||
"state": document["state"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare two self-contained blind CVAT packages for E46."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from k1link.compute.e46_review_handoff import build_e46_review_handoff
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--truth-island-root", type=Path, required=True)
|
||||
parser.add_argument("--evaluation-pack-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = build_e46_review_handoff(
|
||||
truth_island_root=args.truth_island_root,
|
||||
evaluation_pack_root=args.evaluation_pack_root,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result_id": result.result_id,
|
||||
"result_root": str(result.result_root),
|
||||
"state": result.manifest["state"],
|
||||
"frame_count": result.manifest["identity"]["frame_count"],
|
||||
"reviewer_slots": result.manifest["identity"]["reviewer_slots"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,839 @@
|
||||
"""Operational blind-review handoff from E46 into human CVAT review."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .e46_detector_truth_island import (
|
||||
E46_CONTRACT_NAME,
|
||||
E46_MANIFEST_NAME,
|
||||
E46_REFERENCES_NAME,
|
||||
E46_REVIEW_NAME,
|
||||
E46DetectorTruthIslandError,
|
||||
read_e46_detector_truth_island,
|
||||
)
|
||||
from .e48_detector_truth_seal import (
|
||||
E48_REVIEW_SCHEMA,
|
||||
E48DetectorTruthSealError,
|
||||
validate_e48_detector_review_submission,
|
||||
)
|
||||
|
||||
E46_REVIEW_HANDOFF_SCHEMA: Final = "missioncore.e46-review-handoff/v1"
|
||||
E46_REVIEW_HANDOFF_PROFILE: Final = "cvat-coco-empty-blind-two-reviewer/v1"
|
||||
E46_REVIEW_SOURCE_SCHEMA: Final = "missioncore.e46-review-source/v1"
|
||||
E46_REVIEW_HANDOFF_MANIFEST: Final = "manifest.json"
|
||||
E46_REVIEW_HANDOFF_SLOTS: Final = ("reviewer-a", "reviewer-b")
|
||||
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
_BLINDNESS: Final = {
|
||||
"candidate_identity_seen": False,
|
||||
"model_prelabels_seen": False,
|
||||
"model_predictions_seen": False,
|
||||
"model_scores_seen": False,
|
||||
}
|
||||
_SAFE_HANDOFF = re.compile(r"^e46-review-handoff-[a-f0-9]{64}$")
|
||||
_SAFE_REVIEWER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,63}$")
|
||||
_ZIP_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0)
|
||||
|
||||
|
||||
class E46ReviewHandoffError(RuntimeError):
|
||||
"""The E46 reviewer handoff or completed CVAT export is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class E46ReviewHandoff:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
manifest: dict[str, Any]
|
||||
|
||||
|
||||
def build_e46_review_handoff(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
evaluation_pack_root: Path,
|
||||
output_root: Path,
|
||||
) -> E46ReviewHandoff:
|
||||
"""Create two self-contained, prediction-free reviewer packages."""
|
||||
|
||||
try:
|
||||
truth = read_e46_detector_truth_island(truth_island_root)
|
||||
except E46DetectorTruthIslandError as reason:
|
||||
raise E46ReviewHandoffError("E46 truth island is invalid") from reason
|
||||
evaluation_root = evaluation_pack_root.resolve(strict=True)
|
||||
evaluation_manifest = _read_json(evaluation_root / "manifest.json")
|
||||
truth_source = _object(
|
||||
_object(truth.manifest.get("identity"), "E46 identity").get("source"),
|
||||
"E46 source",
|
||||
)
|
||||
if (
|
||||
evaluation_manifest.get("generation_id") != evaluation_root.name
|
||||
or truth_source.get("evaluation_pack_id") != evaluation_root.name
|
||||
):
|
||||
raise E46ReviewHandoffError("E2 evaluation pack identity changed")
|
||||
|
||||
references = tuple(_read_jsonl(truth.result_root / E46_REFERENCES_NAME))
|
||||
contract = _read_json(truth.result_root / E46_CONTRACT_NAME)
|
||||
review_template = _read_json(truth.result_root / E46_REVIEW_NAME)
|
||||
annotation = _object(contract.get("annotation"), "E46 annotation contract")
|
||||
raw_classes = annotation.get("classes")
|
||||
if (
|
||||
contract.get("reviewer_package", {}).get("model_predictions_included")
|
||||
is not False
|
||||
or contract.get("reviewer_package", {}).get("model_prelabels_included")
|
||||
is not False
|
||||
or not isinstance(raw_classes, list)
|
||||
or not raw_classes
|
||||
or not all(isinstance(value, str) and value for value in raw_classes)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 blind annotation contract is invalid")
|
||||
classes = tuple(raw_classes)
|
||||
if (
|
||||
review_template.get("truth_island_id") != truth.result_id
|
||||
or len(_list(review_template.get("images"), "E46 review images"))
|
||||
!= len(references)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 review template is invalid")
|
||||
|
||||
sources = tuple(
|
||||
_verified_source(
|
||||
evaluation_root=evaluation_root,
|
||||
reference=reference,
|
||||
)
|
||||
for reference in references
|
||||
)
|
||||
identity = {
|
||||
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
|
||||
"profile": E46_REVIEW_HANDOFF_PROFILE,
|
||||
"truth_island": {
|
||||
"result_id": truth.result_id,
|
||||
"manifest_sha256": _sha256(truth.result_root / E46_MANIFEST_NAME),
|
||||
"references_sha256": _sha256(
|
||||
truth.result_root / E46_REFERENCES_NAME
|
||||
),
|
||||
},
|
||||
"evaluation_pack": {
|
||||
"generation_id": evaluation_root.name,
|
||||
"manifest_sha256": _sha256(evaluation_root / "manifest.json"),
|
||||
},
|
||||
"reviewer_slots": list(E46_REVIEW_HANDOFF_SLOTS),
|
||||
"frame_count": len(sources),
|
||||
"classes": list(classes),
|
||||
"blindness": _BLINDNESS,
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e46-review-handoff-{identity_sha256}"
|
||||
parent = output_root.expanduser().absolute()
|
||||
destination = parent / result_id
|
||||
if destination.exists():
|
||||
return validate_e46_review_handoff(destination)
|
||||
|
||||
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
staging = parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700)
|
||||
try:
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
_write_reviewer_slot(
|
||||
root=staging / slot,
|
||||
slot=slot,
|
||||
truth_island_id=truth.result_id,
|
||||
sources=sources,
|
||||
classes=classes,
|
||||
review_template=review_template,
|
||||
)
|
||||
_write_text(staging / "README.md", _handoff_readme(truth.result_id))
|
||||
artifacts = [
|
||||
_artifact(path, staging)
|
||||
for path in sorted(staging.rglob("*"))
|
||||
if path.is_file()
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": _utc_now(),
|
||||
"state": "prepared-awaiting-two-independent-human-reviews",
|
||||
"ground_truth": False,
|
||||
"artifacts": artifacts,
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
_write_json(staging / E46_REVIEW_HANDOFF_MANIFEST, manifest)
|
||||
_fsync_tree(staging)
|
||||
os.replace(staging, destination)
|
||||
_fsync_directory(parent)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return validate_e46_review_handoff(destination)
|
||||
|
||||
|
||||
def validate_e46_review_handoff(root: Path) -> E46ReviewHandoff:
|
||||
"""Revalidate a complete E46 blind-review handoff."""
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / E46_REVIEW_HANDOFF_MANIFEST)
|
||||
identity = _object(manifest.get("identity"), "handoff identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
not resolved.is_dir()
|
||||
or _SAFE_HANDOFF.fullmatch(resolved.name) is None
|
||||
or manifest.get("schema_version") != E46_REVIEW_HANDOFF_SCHEMA
|
||||
or manifest.get("result_id") != resolved.name
|
||||
or manifest.get("state")
|
||||
!= "prepared-awaiting-two-independent-human-reviews"
|
||||
or manifest.get("ground_truth") is not False
|
||||
or manifest.get("authority") != _AUTHORITY
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
!= identity_sha256
|
||||
or resolved.name != f"e46-review-handoff-{identity_sha256}"
|
||||
or identity.get("profile") != E46_REVIEW_HANDOFF_PROFILE
|
||||
or identity.get("blindness") != _BLINDNESS
|
||||
or identity.get("reviewer_slots") != list(E46_REVIEW_HANDOFF_SLOTS)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 review handoff identity is invalid")
|
||||
artifacts = _list(manifest.get("artifacts"), "handoff artifacts")
|
||||
artifact_paths: set[str] = set()
|
||||
for raw in artifacts:
|
||||
artifact = _object(raw, "handoff artifact")
|
||||
relative = artifact.get("path")
|
||||
if not isinstance(relative, str):
|
||||
raise E46ReviewHandoffError("handoff artifact path is invalid")
|
||||
path = _safe_relative(resolved, relative)
|
||||
if (
|
||||
not path.is_file()
|
||||
or path.is_symlink()
|
||||
or path.stat().st_size != artifact.get("byte_length")
|
||||
or _sha256(path) != artifact.get("sha256")
|
||||
):
|
||||
raise E46ReviewHandoffError("handoff artifact changed")
|
||||
artifact_paths.add(relative)
|
||||
required = {"README.md"}
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
required.update(
|
||||
{
|
||||
f"{slot}/README.md",
|
||||
f"{slot}/images.zip",
|
||||
f"{slot}/cvat-empty-coco.zip",
|
||||
f"{slot}/labels.json",
|
||||
f"{slot}/source-map.jsonl",
|
||||
f"{slot}/submission-template.json",
|
||||
}
|
||||
)
|
||||
if not required.issubset(artifact_paths):
|
||||
raise E46ReviewHandoffError("reviewer handoff artifacts are incomplete")
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
_validate_slot(
|
||||
resolved / slot,
|
||||
expected_frame_count=_positive_int(
|
||||
identity.get("frame_count"),
|
||||
"handoff frame count",
|
||||
),
|
||||
expected_classes=tuple(
|
||||
_string(value, "handoff class")
|
||||
for value in _list(identity.get("classes"), "handoff classes")
|
||||
),
|
||||
)
|
||||
return E46ReviewHandoff(
|
||||
result_id=resolved.name,
|
||||
result_root=resolved,
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
|
||||
def convert_cvat_coco_to_e48_review(
|
||||
*,
|
||||
handoff_root: Path,
|
||||
truth_island_root: Path,
|
||||
reviewer_slot: str,
|
||||
reviewer_id: str,
|
||||
cvat_export_path: Path,
|
||||
submitted_at_utc: str,
|
||||
output_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Convert one completed CVAT COCO export into a strict E48 review."""
|
||||
|
||||
handoff = validate_e46_review_handoff(handoff_root)
|
||||
if reviewer_slot not in E46_REVIEW_HANDOFF_SLOTS:
|
||||
raise E46ReviewHandoffError("reviewer slot is invalid")
|
||||
if _SAFE_REVIEWER.fullmatch(reviewer_id) is None:
|
||||
raise E46ReviewHandoffError("reviewer identity is invalid")
|
||||
submitted = _utc_timestamp(submitted_at_utc, "review submitted_at_utc")
|
||||
slot_root = handoff.result_root / reviewer_slot
|
||||
source_rows = tuple(_read_jsonl(slot_root / "source-map.jsonl"))
|
||||
template = _read_json(slot_root / "submission-template.json")
|
||||
coco = _read_cvat_coco(cvat_export_path)
|
||||
categories = _coco_categories(
|
||||
coco.get("categories"),
|
||||
allowed=frozenset(handoff.manifest["identity"]["classes"]),
|
||||
)
|
||||
image_names = _coco_images(
|
||||
coco.get("images"),
|
||||
expected={str(row["file_name"]) for row in source_rows},
|
||||
)
|
||||
objects_by_name: dict[str, list[dict[str, Any]]] = {
|
||||
str(row["file_name"]): [] for row in source_rows
|
||||
}
|
||||
raw_annotations = _list(coco.get("annotations"), "COCO annotations")
|
||||
for raw_annotation in raw_annotations:
|
||||
annotation = _object(raw_annotation, "COCO annotation")
|
||||
image_id = annotation.get("image_id")
|
||||
category_id = annotation.get("category_id")
|
||||
if image_id not in image_names or category_id not in categories:
|
||||
raise E46ReviewHandoffError("COCO annotation identity is invalid")
|
||||
attributes = _annotation_attributes(annotation)
|
||||
objects_by_name[image_names[image_id]].append(
|
||||
{
|
||||
"category": categories[category_id],
|
||||
"box_xyxy": _coco_box(annotation.get("bbox")),
|
||||
"occluded": _required_bool(attributes, "occluded"),
|
||||
"truncated": _required_bool(attributes, "truncated"),
|
||||
"notes": _optional_text(attributes.get("notes")),
|
||||
}
|
||||
)
|
||||
images: list[dict[str, Any]] = []
|
||||
template_images = _list(template.get("images"), "submission images")
|
||||
if len(template_images) != len(source_rows):
|
||||
raise E46ReviewHandoffError("submission template coverage changed")
|
||||
for raw_template, source in zip(template_images, source_rows, strict=True):
|
||||
image = dict(_object(raw_template, "submission image"))
|
||||
file_name = str(source["file_name"])
|
||||
raw_objects = sorted(
|
||||
objects_by_name[file_name],
|
||||
key=lambda item: (
|
||||
str(item["category"]),
|
||||
tuple(float(value) for value in item["box_xyxy"]),
|
||||
),
|
||||
)
|
||||
image["review_state"] = "reviewed"
|
||||
image["objects"] = [
|
||||
{"object_id": f"object-{index:03d}", **item}
|
||||
for index, item in enumerate(raw_objects, start=1)
|
||||
]
|
||||
image["hard_negative"] = not raw_objects
|
||||
image["notes"] = None
|
||||
images.append(image)
|
||||
document = {
|
||||
"schema_version": E48_REVIEW_SCHEMA,
|
||||
"truth_island_id": template["truth_island_id"],
|
||||
"state": "completed-independent-no-model-assistance",
|
||||
"reviewer_id": reviewer_id,
|
||||
"review_round": 1,
|
||||
"blindness": _BLINDNESS,
|
||||
"images": images,
|
||||
"acceptance": {
|
||||
"all_images_reviewed": True,
|
||||
"independent": True,
|
||||
"submitted_at_utc": submitted,
|
||||
},
|
||||
}
|
||||
_write_json_atomic(output_path, document)
|
||||
try:
|
||||
validate_e48_detector_review_submission(
|
||||
truth_island_root=truth_island_root,
|
||||
review_path=output_path,
|
||||
)
|
||||
except E48DetectorTruthSealError as reason:
|
||||
raise E46ReviewHandoffError(
|
||||
"converted E48 review is invalid"
|
||||
) from reason
|
||||
return document
|
||||
|
||||
|
||||
def _write_reviewer_slot(
|
||||
*,
|
||||
root: Path,
|
||||
slot: str,
|
||||
truth_island_id: str,
|
||||
sources: tuple[dict[str, Any], ...],
|
||||
classes: tuple[str, ...],
|
||||
review_template: dict[str, Any],
|
||||
) -> None:
|
||||
root.mkdir(mode=0o700, parents=True)
|
||||
file_payloads = tuple(
|
||||
(str(source["file_name"]), Path(source["path"]).read_bytes())
|
||||
for source in sources
|
||||
)
|
||||
_write_zip(root / "images.zip", file_payloads)
|
||||
coco = {
|
||||
"info": {
|
||||
"description": "Mission Core E46 blind detector review",
|
||||
"version": E46_REVIEW_HANDOFF_PROFILE,
|
||||
"truth_island_id": truth_island_id,
|
||||
"reviewer_slot": slot,
|
||||
"model_material_included": False,
|
||||
},
|
||||
"licenses": [],
|
||||
"images": [
|
||||
{
|
||||
"id": int(source["truth_island_sequence"]),
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
"file_name": source["file_name"],
|
||||
}
|
||||
for source in sources
|
||||
],
|
||||
"annotations": [],
|
||||
"categories": [
|
||||
{"id": index, "name": name, "supercategory": "missioncore"}
|
||||
for index, name in enumerate(classes, start=1)
|
||||
],
|
||||
}
|
||||
_write_zip(
|
||||
root / "cvat-empty-coco.zip",
|
||||
(("annotations/instances_default.json", _canonical_json(coco) + b"\n"),),
|
||||
)
|
||||
labels = {
|
||||
"schema_version": "missioncore.e46-cvat-labels/v1",
|
||||
"labels": [
|
||||
{
|
||||
"name": name,
|
||||
"attributes": [
|
||||
{
|
||||
"name": "occluded",
|
||||
"input_type": "checkbox",
|
||||
"default_value": "false",
|
||||
},
|
||||
{
|
||||
"name": "truncated",
|
||||
"input_type": "checkbox",
|
||||
"default_value": "false",
|
||||
},
|
||||
],
|
||||
}
|
||||
for name in classes
|
||||
],
|
||||
}
|
||||
_write_json(root / "labels.json", labels)
|
||||
source_rows = [
|
||||
{
|
||||
"schema_version": E46_REVIEW_SOURCE_SCHEMA,
|
||||
"truth_island_sequence": source["truth_island_sequence"],
|
||||
"file_name": source["file_name"],
|
||||
"source_path": source["source_path"],
|
||||
"source_sha256": source["source_sha256"],
|
||||
"byte_length": source["byte_length"],
|
||||
}
|
||||
for source in sources
|
||||
]
|
||||
_write_jsonl(root / "source-map.jsonl", source_rows)
|
||||
submission = json.loads(json.dumps(review_template))
|
||||
submission["schema_version"] = E48_REVIEW_SCHEMA
|
||||
submission["state"] = "prepared-awaiting-independent-review"
|
||||
submission["reviewer_id"] = None
|
||||
submission["review_round"] = 1
|
||||
submission["blindness"] = _BLINDNESS
|
||||
submission["acceptance"] = None
|
||||
_write_json(root / "submission-template.json", submission)
|
||||
_write_text(root / "README.md", _slot_readme(slot))
|
||||
|
||||
|
||||
def _verified_source(
|
||||
*,
|
||||
evaluation_root: Path,
|
||||
reference: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
relative = _string(reference.get("source_path"), "E46 source path")
|
||||
path = _safe_relative(evaluation_root, relative)
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise E46ReviewHandoffError("E46 source image is unavailable")
|
||||
byte_length = _positive_int(reference.get("byte_length"), "source bytes")
|
||||
source_sha256 = _string(reference.get("sha256"), "source SHA-256")
|
||||
if path.stat().st_size != byte_length or _sha256(path) != source_sha256:
|
||||
raise E46ReviewHandoffError("E46 source image identity changed")
|
||||
with Image.open(path) as image:
|
||||
if image.format != "PNG" or image.size != (800, 600):
|
||||
raise E46ReviewHandoffError("E46 source image geometry changed")
|
||||
image.verify()
|
||||
sequence = _positive_int(
|
||||
reference.get("truth_island_sequence"),
|
||||
"truth island sequence",
|
||||
)
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"file_name": f"e46-{sequence:04d}-{Path(relative).name}",
|
||||
"source_path": relative,
|
||||
"source_sha256": source_sha256,
|
||||
"byte_length": byte_length,
|
||||
"path": path,
|
||||
}
|
||||
|
||||
|
||||
def _validate_slot(
|
||||
root: Path,
|
||||
*,
|
||||
expected_frame_count: int,
|
||||
expected_classes: tuple[str, ...],
|
||||
) -> None:
|
||||
source_rows = tuple(_read_jsonl(root / "source-map.jsonl"))
|
||||
if (
|
||||
len(source_rows) != expected_frame_count
|
||||
or len({row.get("file_name") for row in source_rows})
|
||||
!= expected_frame_count
|
||||
):
|
||||
raise E46ReviewHandoffError("reviewer source map is invalid")
|
||||
with zipfile.ZipFile(root / "images.zip") as archive:
|
||||
names = archive.namelist()
|
||||
if names != [row["file_name"] for row in source_rows]:
|
||||
raise E46ReviewHandoffError("reviewer image archive order changed")
|
||||
for row in source_rows:
|
||||
payload = archive.read(str(row["file_name"]))
|
||||
if (
|
||||
len(payload) != row.get("byte_length")
|
||||
or hashlib.sha256(payload).hexdigest()
|
||||
!= row.get("source_sha256")
|
||||
):
|
||||
raise E46ReviewHandoffError("reviewer image payload changed")
|
||||
coco = _read_zip_json(root / "cvat-empty-coco.zip")
|
||||
if (
|
||||
coco.get("annotations") != []
|
||||
or len(_list(coco.get("images"), "handoff COCO images"))
|
||||
!= expected_frame_count
|
||||
or tuple(
|
||||
_string(row.get("name"), "handoff COCO category")
|
||||
for row in _list(coco.get("categories"), "handoff COCO categories")
|
||||
)
|
||||
!= expected_classes
|
||||
):
|
||||
raise E46ReviewHandoffError("blind CVAT payload changed")
|
||||
|
||||
|
||||
def _read_cvat_coco(path: Path) -> dict[str, Any]:
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.suffix.lower() == ".zip":
|
||||
return _read_zip_json(resolved)
|
||||
return _read_json(resolved)
|
||||
|
||||
|
||||
def _read_zip_json(path: Path) -> dict[str, Any]:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
json_names = [
|
||||
name
|
||||
for name in archive.namelist()
|
||||
if PurePosixPath(name).suffix.lower() == ".json"
|
||||
]
|
||||
if len(json_names) != 1:
|
||||
raise E46ReviewHandoffError(
|
||||
"CVAT COCO archive must contain one JSON document"
|
||||
)
|
||||
try:
|
||||
value = json.loads(archive.read(json_names[0]))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError("CVAT COCO JSON is invalid") from reason
|
||||
return _object(value, "CVAT COCO document")
|
||||
|
||||
|
||||
def _coco_categories(
|
||||
value: object,
|
||||
*,
|
||||
allowed: frozenset[str],
|
||||
) -> dict[object, str]:
|
||||
result: dict[object, str] = {}
|
||||
for raw in _list(value, "COCO categories"):
|
||||
row = _object(raw, "COCO category")
|
||||
category_id = row.get("id")
|
||||
name = row.get("name")
|
||||
if (
|
||||
isinstance(category_id, bool)
|
||||
or not isinstance(category_id, (int, str))
|
||||
or not isinstance(name, str)
|
||||
or name not in allowed
|
||||
or category_id in result
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO category is invalid")
|
||||
result[category_id] = name
|
||||
if not result:
|
||||
raise E46ReviewHandoffError("COCO categories are empty")
|
||||
return result
|
||||
|
||||
|
||||
def _coco_images(
|
||||
value: object,
|
||||
*,
|
||||
expected: set[str],
|
||||
) -> dict[object, str]:
|
||||
result: dict[object, str] = {}
|
||||
observed: set[str] = set()
|
||||
for raw in _list(value, "COCO images"):
|
||||
row = _object(raw, "COCO image")
|
||||
image_id = row.get("id")
|
||||
name = Path(_string(row.get("file_name"), "COCO file name")).name
|
||||
if (
|
||||
isinstance(image_id, bool)
|
||||
or not isinstance(image_id, (int, str))
|
||||
or image_id in result
|
||||
or name in observed
|
||||
or row.get("width") != 800
|
||||
or row.get("height") != 600
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO image is invalid")
|
||||
result[image_id] = name
|
||||
observed.add(name)
|
||||
if observed != expected:
|
||||
raise E46ReviewHandoffError("COCO image coverage is incomplete")
|
||||
return result
|
||||
|
||||
|
||||
def _annotation_attributes(annotation: dict[str, Any]) -> dict[str, Any]:
|
||||
raw = annotation.get("attributes")
|
||||
if isinstance(raw, dict):
|
||||
result = dict(raw)
|
||||
elif isinstance(raw, list):
|
||||
result = {}
|
||||
for item in raw:
|
||||
row = _object(item, "COCO annotation attribute")
|
||||
name = row.get("name")
|
||||
if not isinstance(name, str) or name in result:
|
||||
raise E46ReviewHandoffError("COCO annotation attribute is invalid")
|
||||
result[name] = row.get("value")
|
||||
elif raw is None:
|
||||
result = {}
|
||||
else:
|
||||
raise E46ReviewHandoffError("COCO annotation attributes are invalid")
|
||||
for name in ("occluded", "truncated"):
|
||||
if name not in result and name in annotation:
|
||||
result[name] = annotation[name]
|
||||
return result
|
||||
|
||||
|
||||
def _required_bool(attributes: dict[str, Any], name: str) -> bool:
|
||||
value = attributes.get(name)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str) and value.lower() in {"true", "false"}:
|
||||
return value.lower() == "true"
|
||||
raise E46ReviewHandoffError(f"COCO annotation {name} flag is missing")
|
||||
|
||||
|
||||
def _coco_box(value: object) -> list[float]:
|
||||
raw = _list(value, "COCO bbox")
|
||||
if len(raw) != 4:
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
numbers: list[float] = []
|
||||
for item in raw:
|
||||
if isinstance(item, bool) or not isinstance(item, (int, float)):
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
number = float(item)
|
||||
if not math.isfinite(number):
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
numbers.append(number)
|
||||
x, y, width, height = numbers
|
||||
if (
|
||||
x < 0.0
|
||||
or y < 0.0
|
||||
or width <= 0.0
|
||||
or height <= 0.0
|
||||
or x + width > 800.0
|
||||
or y + height > 600.0
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO bbox is outside the source image")
|
||||
return [x, y, x + width, y + height]
|
||||
|
||||
|
||||
def _safe_relative(root: Path, relative: str) -> Path:
|
||||
pure = PurePosixPath(relative)
|
||||
if pure.is_absolute() or ".." in pure.parts or not pure.parts:
|
||||
raise E46ReviewHandoffError("relative artifact path is unsafe")
|
||||
resolved = (root / Path(*pure.parts)).resolve(strict=True)
|
||||
try:
|
||||
resolved.relative_to(root.resolve(strict=True))
|
||||
except ValueError as reason:
|
||||
raise E46ReviewHandoffError(
|
||||
"relative artifact path escapes its root"
|
||||
) from reason
|
||||
return resolved
|
||||
|
||||
|
||||
def _write_zip(path: Path, entries: tuple[tuple[str, bytes], ...]) -> None:
|
||||
with zipfile.ZipFile(
|
||||
path,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
compresslevel=6,
|
||||
) as archive:
|
||||
for name, payload in entries:
|
||||
info = zipfile.ZipInfo(name, date_time=_ZIP_TIMESTAMP)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o600 << 16
|
||||
archive.writestr(info, payload)
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_json_atomic(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
||||
_write_json(temporary, value)
|
||||
os.replace(temporary, path)
|
||||
_fsync_directory(path.parent)
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_bytes(_canonical_json(value) + b"\n")
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.write_bytes(b"".join(_canonical_json(row) + b"\n" for row in rows))
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_text(path: Path, value: str) -> None:
|
||||
path.write_text(value, encoding="utf-8", newline="\n")
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _artifact(path: Path, root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_bytes())
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError(f"invalid JSON: {path.name}") from reason
|
||||
return _object(value, path.name)
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
rows.append(_object(json.loads(line), path.name))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError(f"invalid JSONL: {path.name}") from reason
|
||||
return rows
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise E46ReviewHandoffError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _list(value: object, label: str) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
raise E46ReviewHandoffError(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
|
||||
def _string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise E46ReviewHandoffError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_int(value: object, label: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise E46ReviewHandoffError(f"{label} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str) or len(value) > 1000:
|
||||
raise E46ReviewHandoffError("COCO annotation notes are invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _utc_timestamp(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise E46ReviewHandoffError(f"{label} is invalid")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as reason:
|
||||
raise E46ReviewHandoffError(f"{label} is invalid") from reason
|
||||
if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed):
|
||||
raise E46ReviewHandoffError(f"{label} must be UTC")
|
||||
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _fsync_tree(root: Path) -> None:
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_file():
|
||||
with path.open("rb") as stream:
|
||||
os.fsync(stream.fileno())
|
||||
elif path.is_dir():
|
||||
_fsync_directory(path)
|
||||
_fsync_directory(root)
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _handoff_readme(truth_island_id: str) -> str:
|
||||
return f"""# Mission Core E46 blind detector review
|
||||
|
||||
Truth island: `{truth_island_id}`
|
||||
|
||||
This handoff contains two equivalent reviewer slots. Give each slot to a
|
||||
different human reviewer. Reviewers must not see E47 predictions, scores,
|
||||
candidate identities, model prelabels or each other's annotations.
|
||||
|
||||
After both independent reviews are complete, convert each CVAT COCO export with
|
||||
`experiments/perception/convert_e46_cvat_review.py`. Adjudication and E48 sealing
|
||||
happen only after both converted submissions pass validation.
|
||||
"""
|
||||
|
||||
|
||||
def _slot_readme(slot: str) -> str:
|
||||
return f"""# E46 blind review slot: {slot}
|
||||
|
||||
1. Create a new CVAT image task with the seven labels in `labels.json`.
|
||||
2. Add boolean `occluded` and `truncated` attributes to every label.
|
||||
3. Upload `images.zip` as task data.
|
||||
4. Import `cvat-empty-coco.zip` as COCO 1.0 annotations. It intentionally
|
||||
contains zero objects and no model material.
|
||||
5. Draw every identifiable in-FOV instance. Frames with no objects remain empty.
|
||||
6. Set both attributes explicitly for every object.
|
||||
7. Export the completed task as COCO 1.0.
|
||||
|
||||
Do not use model assistance, prelabels, E47 predictions or another review.
|
||||
"""
|
||||
@@ -55,6 +55,36 @@ class E48DetectorTruthSealError(RuntimeError):
|
||||
"""An independent review or truth-seal artifact is invalid."""
|
||||
|
||||
|
||||
def validate_e48_detector_review_submission(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
review_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate one completed blind review before adjudication."""
|
||||
|
||||
try:
|
||||
truth_island = read_e46_detector_truth_island(truth_island_root)
|
||||
except E46DetectorTruthIslandError as reason:
|
||||
raise E48DetectorTruthSealError("E46 truth island is invalid") from reason
|
||||
references = tuple(
|
||||
_read_jsonl(truth_island.result_root / E46_REFERENCES_NAME)
|
||||
)
|
||||
contract = _read_json(truth_island.result_root / E46_CONTRACT_NAME)
|
||||
annotation = _object(contract.get("annotation"), "E46 annotation contract")
|
||||
raw_classes = annotation.get("classes")
|
||||
if not isinstance(raw_classes, list) or not all(
|
||||
isinstance(value, str) for value in raw_classes
|
||||
):
|
||||
raise E48DetectorTruthSealError("E46 target classes are invalid")
|
||||
document = _read_json(review_path.resolve(strict=True))
|
||||
return _validate_review(
|
||||
document,
|
||||
truth_island_id=truth_island.result_id,
|
||||
references=references,
|
||||
target_classes=frozenset(raw_classes),
|
||||
)
|
||||
|
||||
|
||||
def build_e48_detector_truth_seal(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from k1link.compute import e46_review_handoff as handoff
|
||||
from k1link.compute import e48_detector_truth_seal as e48
|
||||
|
||||
|
||||
def _canonical(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(_canonical(value) + b"\n")
|
||||
|
||||
|
||||
def _fixture(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> tuple[Path, Path, SimpleNamespace]:
|
||||
truth_root = tmp_path / ("e46-detector-truth-island-" + "a" * 64)
|
||||
evaluation_root = tmp_path / ("evaluation-pack-" + "b" * 64)
|
||||
truth_root.mkdir()
|
||||
image_root = evaluation_root / "images" / "valid-fov-fill"
|
||||
image_root.mkdir(parents=True)
|
||||
references: list[dict[str, Any]] = []
|
||||
review_images: list[dict[str, Any]] = []
|
||||
for sequence, image_id in enumerate((2, 5), start=1):
|
||||
frame_index = sequence * 10
|
||||
name = f"image-{image_id:03d}-frame-{frame_index:06d}.png"
|
||||
path = image_root / name
|
||||
Image.new("RGB", (800, 600), (sequence * 30, 10, 20)).save(path)
|
||||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
relative = f"images/valid-fov-fill/{name}"
|
||||
reference = {
|
||||
"schema_version": "missioncore.e46-truth-island-image-reference/v1",
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": image_id,
|
||||
"frame_index": frame_index,
|
||||
"session_seconds": float(sequence),
|
||||
"role": "anchor",
|
||||
"group_id": f"anchor-{sequence}",
|
||||
"source_path": relative,
|
||||
"sha256": digest,
|
||||
"byte_length": path.stat().st_size,
|
||||
}
|
||||
references.append(reference)
|
||||
review_images.append(
|
||||
{
|
||||
"truth_island_sequence": sequence,
|
||||
"image_id": image_id,
|
||||
"frame_index": frame_index,
|
||||
"session_seconds": float(sequence),
|
||||
"role": "anchor",
|
||||
"group_id": f"anchor-{sequence}",
|
||||
"source_path": relative,
|
||||
"source_sha256": digest,
|
||||
"review_state": "pending",
|
||||
"hard_negative": None,
|
||||
"objects": [],
|
||||
"notes": None,
|
||||
}
|
||||
)
|
||||
_write_json(
|
||||
evaluation_root / "manifest.json",
|
||||
{"generation_id": evaluation_root.name},
|
||||
)
|
||||
(truth_root / "image-references.jsonl").write_bytes(
|
||||
b"".join(_canonical(row) + b"\n" for row in references)
|
||||
)
|
||||
_write_json(
|
||||
truth_root / "blind-contract.json",
|
||||
{
|
||||
"annotation": {"classes": ["person", "car"]},
|
||||
"reviewer_package": {
|
||||
"model_predictions_included": False,
|
||||
"model_prelabels_included": False,
|
||||
},
|
||||
},
|
||||
)
|
||||
_write_json(
|
||||
truth_root / "review-template.json",
|
||||
{
|
||||
"schema_version": "missioncore.e46-detector-review-template/v1",
|
||||
"truth_island_id": truth_root.name,
|
||||
"state": "prepared-unreviewed-no-prelabels",
|
||||
"reviewer_id": None,
|
||||
"review_round": None,
|
||||
"images": review_images,
|
||||
"acceptance": None,
|
||||
},
|
||||
)
|
||||
manifest = {
|
||||
"identity": {
|
||||
"source": {"evaluation_pack_id": evaluation_root.name},
|
||||
}
|
||||
}
|
||||
_write_json(truth_root / "manifest.json", manifest)
|
||||
truth = SimpleNamespace(
|
||||
result_id=truth_root.name,
|
||||
result_root=truth_root,
|
||||
manifest=manifest,
|
||||
report={"status": "prepared-awaiting-independent-human-review"},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
handoff,
|
||||
"read_e46_detector_truth_island",
|
||||
lambda _root: truth,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
e48,
|
||||
"read_e46_detector_truth_island",
|
||||
lambda _root: truth,
|
||||
)
|
||||
return truth_root, evaluation_root, truth
|
||||
|
||||
|
||||
def test_handoff_is_blind_reproducible_and_converts_cvat(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
|
||||
result = handoff.build_e46_review_handoff(
|
||||
truth_island_root=truth_root,
|
||||
evaluation_pack_root=evaluation_root,
|
||||
output_root=tmp_path / "handoffs",
|
||||
)
|
||||
repeated = handoff.build_e46_review_handoff(
|
||||
truth_island_root=truth_root,
|
||||
evaluation_pack_root=evaluation_root,
|
||||
output_root=tmp_path / "handoffs",
|
||||
)
|
||||
|
||||
assert repeated.result_id == result.result_id
|
||||
assert result.manifest["identity"]["frame_count"] == 2
|
||||
with zipfile.ZipFile(
|
||||
result.result_root / "reviewer-a" / "cvat-empty-coco.zip"
|
||||
) as archive:
|
||||
coco = json.loads(archive.read("annotations/instances_default.json"))
|
||||
assert coco["annotations"] == []
|
||||
assert "predictions" not in json.dumps(coco).lower()
|
||||
|
||||
source_rows = [
|
||||
json.loads(line)
|
||||
for line in (
|
||||
result.result_root / "reviewer-a" / "source-map.jsonl"
|
||||
).read_text().splitlines()
|
||||
]
|
||||
cvat = {
|
||||
"images": [
|
||||
{
|
||||
"id": index,
|
||||
"file_name": row["file_name"],
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
}
|
||||
for index, row in enumerate(source_rows, start=11)
|
||||
],
|
||||
"categories": [
|
||||
{"id": 1, "name": "person"},
|
||||
{"id": 2, "name": "car"},
|
||||
],
|
||||
"annotations": [
|
||||
{
|
||||
"id": 1,
|
||||
"image_id": 11,
|
||||
"category_id": 2,
|
||||
"bbox": [10.0, 20.0, 100.0, 200.0],
|
||||
"attributes": {
|
||||
"occluded": False,
|
||||
"truncated": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
cvat_path = tmp_path / "review-a.json"
|
||||
_write_json(cvat_path, cvat)
|
||||
output_path = tmp_path / "e48-review-a.json"
|
||||
document = handoff.convert_cvat_coco_to_e48_review(
|
||||
handoff_root=result.result_root,
|
||||
truth_island_root=truth_root,
|
||||
reviewer_slot="reviewer-a",
|
||||
reviewer_id="human-reviewer-a",
|
||||
cvat_export_path=cvat_path,
|
||||
submitted_at_utc="2026-07-29T20:00:00Z",
|
||||
output_path=output_path,
|
||||
)
|
||||
|
||||
assert document["state"] == "completed-independent-no-model-assistance"
|
||||
assert document["images"][0]["objects"][0]["box_xyxy"] == [
|
||||
10.0,
|
||||
20.0,
|
||||
110.0,
|
||||
220.0,
|
||||
]
|
||||
assert document["images"][1]["hard_negative"] is True
|
||||
assert e48.validate_e48_detector_review_submission(
|
||||
truth_island_root=truth_root,
|
||||
review_path=output_path,
|
||||
)["reviewer_id"] == "human-reviewer-a"
|
||||
|
||||
|
||||
def test_handoff_rejects_changed_source_image(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
|
||||
source = next(
|
||||
(evaluation_root / "images" / "valid-fov-fill").glob("*.png")
|
||||
)
|
||||
source.write_bytes(b"changed")
|
||||
|
||||
with pytest.raises(handoff.E46ReviewHandoffError, match="identity changed"):
|
||||
handoff.build_e46_review_handoff(
|
||||
truth_island_root=truth_root,
|
||||
evaluation_pack_root=evaluation_root,
|
||||
output_root=tmp_path / "handoffs",
|
||||
)
|
||||
|
||||
|
||||
def test_converter_requires_explicit_occlusion_and_truncation(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
|
||||
result = handoff.build_e46_review_handoff(
|
||||
truth_island_root=truth_root,
|
||||
evaluation_pack_root=evaluation_root,
|
||||
output_root=tmp_path / "handoffs",
|
||||
)
|
||||
source_rows = [
|
||||
json.loads(line)
|
||||
for line in (
|
||||
result.result_root / "reviewer-a" / "source-map.jsonl"
|
||||
).read_text().splitlines()
|
||||
]
|
||||
cvat = {
|
||||
"images": [
|
||||
{
|
||||
"id": index,
|
||||
"file_name": row["file_name"],
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
}
|
||||
for index, row in enumerate(source_rows, start=1)
|
||||
],
|
||||
"categories": [{"id": 1, "name": "car"}],
|
||||
"annotations": [
|
||||
{
|
||||
"id": 1,
|
||||
"image_id": 1,
|
||||
"category_id": 1,
|
||||
"bbox": [10.0, 20.0, 100.0, 200.0],
|
||||
}
|
||||
],
|
||||
}
|
||||
cvat_path = tmp_path / "invalid.json"
|
||||
_write_json(cvat_path, cvat)
|
||||
|
||||
with pytest.raises(
|
||||
handoff.E46ReviewHandoffError,
|
||||
match="occluded flag is missing",
|
||||
):
|
||||
handoff.convert_cvat_coco_to_e48_review(
|
||||
handoff_root=result.result_root,
|
||||
truth_island_root=truth_root,
|
||||
reviewer_slot="reviewer-a",
|
||||
reviewer_id="human-reviewer-a",
|
||||
cvat_export_path=cvat_path,
|
||||
submitted_at_utc="2026-07-29T20:00:00Z",
|
||||
output_path=tmp_path / "invalid-review.json",
|
||||
)
|
||||
Reference in New Issue
Block a user