feat(perception): prepare blind detector review handoff
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,839 @@
|
||||
"""Operational blind-review handoff from E46 into human CVAT review."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from .e46_detector_truth_island import (
|
||||
E46_CONTRACT_NAME,
|
||||
E46_MANIFEST_NAME,
|
||||
E46_REFERENCES_NAME,
|
||||
E46_REVIEW_NAME,
|
||||
E46DetectorTruthIslandError,
|
||||
read_e46_detector_truth_island,
|
||||
)
|
||||
from .e48_detector_truth_seal import (
|
||||
E48_REVIEW_SCHEMA,
|
||||
E48DetectorTruthSealError,
|
||||
validate_e48_detector_review_submission,
|
||||
)
|
||||
|
||||
E46_REVIEW_HANDOFF_SCHEMA: Final = "missioncore.e46-review-handoff/v1"
|
||||
E46_REVIEW_HANDOFF_PROFILE: Final = "cvat-coco-empty-blind-two-reviewer/v1"
|
||||
E46_REVIEW_SOURCE_SCHEMA: Final = "missioncore.e46-review-source/v1"
|
||||
E46_REVIEW_HANDOFF_MANIFEST: Final = "manifest.json"
|
||||
E46_REVIEW_HANDOFF_SLOTS: Final = ("reviewer-a", "reviewer-b")
|
||||
|
||||
_AUTHORITY: Final = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
}
|
||||
_BLINDNESS: Final = {
|
||||
"candidate_identity_seen": False,
|
||||
"model_prelabels_seen": False,
|
||||
"model_predictions_seen": False,
|
||||
"model_scores_seen": False,
|
||||
}
|
||||
_SAFE_HANDOFF = re.compile(r"^e46-review-handoff-[a-f0-9]{64}$")
|
||||
_SAFE_REVIEWER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,63}$")
|
||||
_ZIP_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0)
|
||||
|
||||
|
||||
class E46ReviewHandoffError(RuntimeError):
|
||||
"""The E46 reviewer handoff or completed CVAT export is invalid."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class E46ReviewHandoff:
|
||||
result_id: str
|
||||
result_root: Path
|
||||
manifest: dict[str, Any]
|
||||
|
||||
|
||||
def build_e46_review_handoff(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
evaluation_pack_root: Path,
|
||||
output_root: Path,
|
||||
) -> E46ReviewHandoff:
|
||||
"""Create two self-contained, prediction-free reviewer packages."""
|
||||
|
||||
try:
|
||||
truth = read_e46_detector_truth_island(truth_island_root)
|
||||
except E46DetectorTruthIslandError as reason:
|
||||
raise E46ReviewHandoffError("E46 truth island is invalid") from reason
|
||||
evaluation_root = evaluation_pack_root.resolve(strict=True)
|
||||
evaluation_manifest = _read_json(evaluation_root / "manifest.json")
|
||||
truth_source = _object(
|
||||
_object(truth.manifest.get("identity"), "E46 identity").get("source"),
|
||||
"E46 source",
|
||||
)
|
||||
if (
|
||||
evaluation_manifest.get("generation_id") != evaluation_root.name
|
||||
or truth_source.get("evaluation_pack_id") != evaluation_root.name
|
||||
):
|
||||
raise E46ReviewHandoffError("E2 evaluation pack identity changed")
|
||||
|
||||
references = tuple(_read_jsonl(truth.result_root / E46_REFERENCES_NAME))
|
||||
contract = _read_json(truth.result_root / E46_CONTRACT_NAME)
|
||||
review_template = _read_json(truth.result_root / E46_REVIEW_NAME)
|
||||
annotation = _object(contract.get("annotation"), "E46 annotation contract")
|
||||
raw_classes = annotation.get("classes")
|
||||
if (
|
||||
contract.get("reviewer_package", {}).get("model_predictions_included")
|
||||
is not False
|
||||
or contract.get("reviewer_package", {}).get("model_prelabels_included")
|
||||
is not False
|
||||
or not isinstance(raw_classes, list)
|
||||
or not raw_classes
|
||||
or not all(isinstance(value, str) and value for value in raw_classes)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 blind annotation contract is invalid")
|
||||
classes = tuple(raw_classes)
|
||||
if (
|
||||
review_template.get("truth_island_id") != truth.result_id
|
||||
or len(_list(review_template.get("images"), "E46 review images"))
|
||||
!= len(references)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 review template is invalid")
|
||||
|
||||
sources = tuple(
|
||||
_verified_source(
|
||||
evaluation_root=evaluation_root,
|
||||
reference=reference,
|
||||
)
|
||||
for reference in references
|
||||
)
|
||||
identity = {
|
||||
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
|
||||
"profile": E46_REVIEW_HANDOFF_PROFILE,
|
||||
"truth_island": {
|
||||
"result_id": truth.result_id,
|
||||
"manifest_sha256": _sha256(truth.result_root / E46_MANIFEST_NAME),
|
||||
"references_sha256": _sha256(
|
||||
truth.result_root / E46_REFERENCES_NAME
|
||||
),
|
||||
},
|
||||
"evaluation_pack": {
|
||||
"generation_id": evaluation_root.name,
|
||||
"manifest_sha256": _sha256(evaluation_root / "manifest.json"),
|
||||
},
|
||||
"reviewer_slots": list(E46_REVIEW_HANDOFF_SLOTS),
|
||||
"frame_count": len(sources),
|
||||
"classes": list(classes),
|
||||
"blindness": _BLINDNESS,
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
result_id = f"e46-review-handoff-{identity_sha256}"
|
||||
parent = output_root.expanduser().absolute()
|
||||
destination = parent / result_id
|
||||
if destination.exists():
|
||||
return validate_e46_review_handoff(destination)
|
||||
|
||||
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
staging = parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
|
||||
staging.mkdir(mode=0o700)
|
||||
try:
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
_write_reviewer_slot(
|
||||
root=staging / slot,
|
||||
slot=slot,
|
||||
truth_island_id=truth.result_id,
|
||||
sources=sources,
|
||||
classes=classes,
|
||||
review_template=review_template,
|
||||
)
|
||||
_write_text(staging / "README.md", _handoff_readme(truth.result_id))
|
||||
artifacts = [
|
||||
_artifact(path, staging)
|
||||
for path in sorted(staging.rglob("*"))
|
||||
if path.is_file()
|
||||
]
|
||||
manifest = {
|
||||
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"identity": identity,
|
||||
"created_at_utc": _utc_now(),
|
||||
"state": "prepared-awaiting-two-independent-human-reviews",
|
||||
"ground_truth": False,
|
||||
"artifacts": artifacts,
|
||||
"authority": _AUTHORITY,
|
||||
}
|
||||
_write_json(staging / E46_REVIEW_HANDOFF_MANIFEST, manifest)
|
||||
_fsync_tree(staging)
|
||||
os.replace(staging, destination)
|
||||
_fsync_directory(parent)
|
||||
except BaseException:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
return validate_e46_review_handoff(destination)
|
||||
|
||||
|
||||
def validate_e46_review_handoff(root: Path) -> E46ReviewHandoff:
|
||||
"""Revalidate a complete E46 blind-review handoff."""
|
||||
|
||||
resolved = root.resolve(strict=True)
|
||||
manifest = _read_json(resolved / E46_REVIEW_HANDOFF_MANIFEST)
|
||||
identity = _object(manifest.get("identity"), "handoff identity")
|
||||
identity_sha256 = manifest.get("identity_sha256")
|
||||
if (
|
||||
not resolved.is_dir()
|
||||
or _SAFE_HANDOFF.fullmatch(resolved.name) is None
|
||||
or manifest.get("schema_version") != E46_REVIEW_HANDOFF_SCHEMA
|
||||
or manifest.get("result_id") != resolved.name
|
||||
or manifest.get("state")
|
||||
!= "prepared-awaiting-two-independent-human-reviews"
|
||||
or manifest.get("ground_truth") is not False
|
||||
or manifest.get("authority") != _AUTHORITY
|
||||
or not isinstance(identity_sha256, str)
|
||||
or hashlib.sha256(_canonical_json(identity)).hexdigest()
|
||||
!= identity_sha256
|
||||
or resolved.name != f"e46-review-handoff-{identity_sha256}"
|
||||
or identity.get("profile") != E46_REVIEW_HANDOFF_PROFILE
|
||||
or identity.get("blindness") != _BLINDNESS
|
||||
or identity.get("reviewer_slots") != list(E46_REVIEW_HANDOFF_SLOTS)
|
||||
):
|
||||
raise E46ReviewHandoffError("E46 review handoff identity is invalid")
|
||||
artifacts = _list(manifest.get("artifacts"), "handoff artifacts")
|
||||
artifact_paths: set[str] = set()
|
||||
for raw in artifacts:
|
||||
artifact = _object(raw, "handoff artifact")
|
||||
relative = artifact.get("path")
|
||||
if not isinstance(relative, str):
|
||||
raise E46ReviewHandoffError("handoff artifact path is invalid")
|
||||
path = _safe_relative(resolved, relative)
|
||||
if (
|
||||
not path.is_file()
|
||||
or path.is_symlink()
|
||||
or path.stat().st_size != artifact.get("byte_length")
|
||||
or _sha256(path) != artifact.get("sha256")
|
||||
):
|
||||
raise E46ReviewHandoffError("handoff artifact changed")
|
||||
artifact_paths.add(relative)
|
||||
required = {"README.md"}
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
required.update(
|
||||
{
|
||||
f"{slot}/README.md",
|
||||
f"{slot}/images.zip",
|
||||
f"{slot}/cvat-empty-coco.zip",
|
||||
f"{slot}/labels.json",
|
||||
f"{slot}/source-map.jsonl",
|
||||
f"{slot}/submission-template.json",
|
||||
}
|
||||
)
|
||||
if not required.issubset(artifact_paths):
|
||||
raise E46ReviewHandoffError("reviewer handoff artifacts are incomplete")
|
||||
for slot in E46_REVIEW_HANDOFF_SLOTS:
|
||||
_validate_slot(
|
||||
resolved / slot,
|
||||
expected_frame_count=_positive_int(
|
||||
identity.get("frame_count"),
|
||||
"handoff frame count",
|
||||
),
|
||||
expected_classes=tuple(
|
||||
_string(value, "handoff class")
|
||||
for value in _list(identity.get("classes"), "handoff classes")
|
||||
),
|
||||
)
|
||||
return E46ReviewHandoff(
|
||||
result_id=resolved.name,
|
||||
result_root=resolved,
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
|
||||
def convert_cvat_coco_to_e48_review(
|
||||
*,
|
||||
handoff_root: Path,
|
||||
truth_island_root: Path,
|
||||
reviewer_slot: str,
|
||||
reviewer_id: str,
|
||||
cvat_export_path: Path,
|
||||
submitted_at_utc: str,
|
||||
output_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Convert one completed CVAT COCO export into a strict E48 review."""
|
||||
|
||||
handoff = validate_e46_review_handoff(handoff_root)
|
||||
if reviewer_slot not in E46_REVIEW_HANDOFF_SLOTS:
|
||||
raise E46ReviewHandoffError("reviewer slot is invalid")
|
||||
if _SAFE_REVIEWER.fullmatch(reviewer_id) is None:
|
||||
raise E46ReviewHandoffError("reviewer identity is invalid")
|
||||
submitted = _utc_timestamp(submitted_at_utc, "review submitted_at_utc")
|
||||
slot_root = handoff.result_root / reviewer_slot
|
||||
source_rows = tuple(_read_jsonl(slot_root / "source-map.jsonl"))
|
||||
template = _read_json(slot_root / "submission-template.json")
|
||||
coco = _read_cvat_coco(cvat_export_path)
|
||||
categories = _coco_categories(
|
||||
coco.get("categories"),
|
||||
allowed=frozenset(handoff.manifest["identity"]["classes"]),
|
||||
)
|
||||
image_names = _coco_images(
|
||||
coco.get("images"),
|
||||
expected={str(row["file_name"]) for row in source_rows},
|
||||
)
|
||||
objects_by_name: dict[str, list[dict[str, Any]]] = {
|
||||
str(row["file_name"]): [] for row in source_rows
|
||||
}
|
||||
raw_annotations = _list(coco.get("annotations"), "COCO annotations")
|
||||
for raw_annotation in raw_annotations:
|
||||
annotation = _object(raw_annotation, "COCO annotation")
|
||||
image_id = annotation.get("image_id")
|
||||
category_id = annotation.get("category_id")
|
||||
if image_id not in image_names or category_id not in categories:
|
||||
raise E46ReviewHandoffError("COCO annotation identity is invalid")
|
||||
attributes = _annotation_attributes(annotation)
|
||||
objects_by_name[image_names[image_id]].append(
|
||||
{
|
||||
"category": categories[category_id],
|
||||
"box_xyxy": _coco_box(annotation.get("bbox")),
|
||||
"occluded": _required_bool(attributes, "occluded"),
|
||||
"truncated": _required_bool(attributes, "truncated"),
|
||||
"notes": _optional_text(attributes.get("notes")),
|
||||
}
|
||||
)
|
||||
images: list[dict[str, Any]] = []
|
||||
template_images = _list(template.get("images"), "submission images")
|
||||
if len(template_images) != len(source_rows):
|
||||
raise E46ReviewHandoffError("submission template coverage changed")
|
||||
for raw_template, source in zip(template_images, source_rows, strict=True):
|
||||
image = dict(_object(raw_template, "submission image"))
|
||||
file_name = str(source["file_name"])
|
||||
raw_objects = sorted(
|
||||
objects_by_name[file_name],
|
||||
key=lambda item: (
|
||||
str(item["category"]),
|
||||
tuple(float(value) for value in item["box_xyxy"]),
|
||||
),
|
||||
)
|
||||
image["review_state"] = "reviewed"
|
||||
image["objects"] = [
|
||||
{"object_id": f"object-{index:03d}", **item}
|
||||
for index, item in enumerate(raw_objects, start=1)
|
||||
]
|
||||
image["hard_negative"] = not raw_objects
|
||||
image["notes"] = None
|
||||
images.append(image)
|
||||
document = {
|
||||
"schema_version": E48_REVIEW_SCHEMA,
|
||||
"truth_island_id": template["truth_island_id"],
|
||||
"state": "completed-independent-no-model-assistance",
|
||||
"reviewer_id": reviewer_id,
|
||||
"review_round": 1,
|
||||
"blindness": _BLINDNESS,
|
||||
"images": images,
|
||||
"acceptance": {
|
||||
"all_images_reviewed": True,
|
||||
"independent": True,
|
||||
"submitted_at_utc": submitted,
|
||||
},
|
||||
}
|
||||
_write_json_atomic(output_path, document)
|
||||
try:
|
||||
validate_e48_detector_review_submission(
|
||||
truth_island_root=truth_island_root,
|
||||
review_path=output_path,
|
||||
)
|
||||
except E48DetectorTruthSealError as reason:
|
||||
raise E46ReviewHandoffError(
|
||||
"converted E48 review is invalid"
|
||||
) from reason
|
||||
return document
|
||||
|
||||
|
||||
def _write_reviewer_slot(
|
||||
*,
|
||||
root: Path,
|
||||
slot: str,
|
||||
truth_island_id: str,
|
||||
sources: tuple[dict[str, Any], ...],
|
||||
classes: tuple[str, ...],
|
||||
review_template: dict[str, Any],
|
||||
) -> None:
|
||||
root.mkdir(mode=0o700, parents=True)
|
||||
file_payloads = tuple(
|
||||
(str(source["file_name"]), Path(source["path"]).read_bytes())
|
||||
for source in sources
|
||||
)
|
||||
_write_zip(root / "images.zip", file_payloads)
|
||||
coco = {
|
||||
"info": {
|
||||
"description": "Mission Core E46 blind detector review",
|
||||
"version": E46_REVIEW_HANDOFF_PROFILE,
|
||||
"truth_island_id": truth_island_id,
|
||||
"reviewer_slot": slot,
|
||||
"model_material_included": False,
|
||||
},
|
||||
"licenses": [],
|
||||
"images": [
|
||||
{
|
||||
"id": int(source["truth_island_sequence"]),
|
||||
"width": 800,
|
||||
"height": 600,
|
||||
"file_name": source["file_name"],
|
||||
}
|
||||
for source in sources
|
||||
],
|
||||
"annotations": [],
|
||||
"categories": [
|
||||
{"id": index, "name": name, "supercategory": "missioncore"}
|
||||
for index, name in enumerate(classes, start=1)
|
||||
],
|
||||
}
|
||||
_write_zip(
|
||||
root / "cvat-empty-coco.zip",
|
||||
(("annotations/instances_default.json", _canonical_json(coco) + b"\n"),),
|
||||
)
|
||||
labels = {
|
||||
"schema_version": "missioncore.e46-cvat-labels/v1",
|
||||
"labels": [
|
||||
{
|
||||
"name": name,
|
||||
"attributes": [
|
||||
{
|
||||
"name": "occluded",
|
||||
"input_type": "checkbox",
|
||||
"default_value": "false",
|
||||
},
|
||||
{
|
||||
"name": "truncated",
|
||||
"input_type": "checkbox",
|
||||
"default_value": "false",
|
||||
},
|
||||
],
|
||||
}
|
||||
for name in classes
|
||||
],
|
||||
}
|
||||
_write_json(root / "labels.json", labels)
|
||||
source_rows = [
|
||||
{
|
||||
"schema_version": E46_REVIEW_SOURCE_SCHEMA,
|
||||
"truth_island_sequence": source["truth_island_sequence"],
|
||||
"file_name": source["file_name"],
|
||||
"source_path": source["source_path"],
|
||||
"source_sha256": source["source_sha256"],
|
||||
"byte_length": source["byte_length"],
|
||||
}
|
||||
for source in sources
|
||||
]
|
||||
_write_jsonl(root / "source-map.jsonl", source_rows)
|
||||
submission = json.loads(json.dumps(review_template))
|
||||
submission["schema_version"] = E48_REVIEW_SCHEMA
|
||||
submission["state"] = "prepared-awaiting-independent-review"
|
||||
submission["reviewer_id"] = None
|
||||
submission["review_round"] = 1
|
||||
submission["blindness"] = _BLINDNESS
|
||||
submission["acceptance"] = None
|
||||
_write_json(root / "submission-template.json", submission)
|
||||
_write_text(root / "README.md", _slot_readme(slot))
|
||||
|
||||
|
||||
def _verified_source(
|
||||
*,
|
||||
evaluation_root: Path,
|
||||
reference: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
relative = _string(reference.get("source_path"), "E46 source path")
|
||||
path = _safe_relative(evaluation_root, relative)
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise E46ReviewHandoffError("E46 source image is unavailable")
|
||||
byte_length = _positive_int(reference.get("byte_length"), "source bytes")
|
||||
source_sha256 = _string(reference.get("sha256"), "source SHA-256")
|
||||
if path.stat().st_size != byte_length or _sha256(path) != source_sha256:
|
||||
raise E46ReviewHandoffError("E46 source image identity changed")
|
||||
with Image.open(path) as image:
|
||||
if image.format != "PNG" or image.size != (800, 600):
|
||||
raise E46ReviewHandoffError("E46 source image geometry changed")
|
||||
image.verify()
|
||||
sequence = _positive_int(
|
||||
reference.get("truth_island_sequence"),
|
||||
"truth island sequence",
|
||||
)
|
||||
return {
|
||||
"truth_island_sequence": sequence,
|
||||
"file_name": f"e46-{sequence:04d}-{Path(relative).name}",
|
||||
"source_path": relative,
|
||||
"source_sha256": source_sha256,
|
||||
"byte_length": byte_length,
|
||||
"path": path,
|
||||
}
|
||||
|
||||
|
||||
def _validate_slot(
|
||||
root: Path,
|
||||
*,
|
||||
expected_frame_count: int,
|
||||
expected_classes: tuple[str, ...],
|
||||
) -> None:
|
||||
source_rows = tuple(_read_jsonl(root / "source-map.jsonl"))
|
||||
if (
|
||||
len(source_rows) != expected_frame_count
|
||||
or len({row.get("file_name") for row in source_rows})
|
||||
!= expected_frame_count
|
||||
):
|
||||
raise E46ReviewHandoffError("reviewer source map is invalid")
|
||||
with zipfile.ZipFile(root / "images.zip") as archive:
|
||||
names = archive.namelist()
|
||||
if names != [row["file_name"] for row in source_rows]:
|
||||
raise E46ReviewHandoffError("reviewer image archive order changed")
|
||||
for row in source_rows:
|
||||
payload = archive.read(str(row["file_name"]))
|
||||
if (
|
||||
len(payload) != row.get("byte_length")
|
||||
or hashlib.sha256(payload).hexdigest()
|
||||
!= row.get("source_sha256")
|
||||
):
|
||||
raise E46ReviewHandoffError("reviewer image payload changed")
|
||||
coco = _read_zip_json(root / "cvat-empty-coco.zip")
|
||||
if (
|
||||
coco.get("annotations") != []
|
||||
or len(_list(coco.get("images"), "handoff COCO images"))
|
||||
!= expected_frame_count
|
||||
or tuple(
|
||||
_string(row.get("name"), "handoff COCO category")
|
||||
for row in _list(coco.get("categories"), "handoff COCO categories")
|
||||
)
|
||||
!= expected_classes
|
||||
):
|
||||
raise E46ReviewHandoffError("blind CVAT payload changed")
|
||||
|
||||
|
||||
def _read_cvat_coco(path: Path) -> dict[str, Any]:
|
||||
resolved = path.resolve(strict=True)
|
||||
if resolved.suffix.lower() == ".zip":
|
||||
return _read_zip_json(resolved)
|
||||
return _read_json(resolved)
|
||||
|
||||
|
||||
def _read_zip_json(path: Path) -> dict[str, Any]:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
json_names = [
|
||||
name
|
||||
for name in archive.namelist()
|
||||
if PurePosixPath(name).suffix.lower() == ".json"
|
||||
]
|
||||
if len(json_names) != 1:
|
||||
raise E46ReviewHandoffError(
|
||||
"CVAT COCO archive must contain one JSON document"
|
||||
)
|
||||
try:
|
||||
value = json.loads(archive.read(json_names[0]))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError("CVAT COCO JSON is invalid") from reason
|
||||
return _object(value, "CVAT COCO document")
|
||||
|
||||
|
||||
def _coco_categories(
|
||||
value: object,
|
||||
*,
|
||||
allowed: frozenset[str],
|
||||
) -> dict[object, str]:
|
||||
result: dict[object, str] = {}
|
||||
for raw in _list(value, "COCO categories"):
|
||||
row = _object(raw, "COCO category")
|
||||
category_id = row.get("id")
|
||||
name = row.get("name")
|
||||
if (
|
||||
isinstance(category_id, bool)
|
||||
or not isinstance(category_id, (int, str))
|
||||
or not isinstance(name, str)
|
||||
or name not in allowed
|
||||
or category_id in result
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO category is invalid")
|
||||
result[category_id] = name
|
||||
if not result:
|
||||
raise E46ReviewHandoffError("COCO categories are empty")
|
||||
return result
|
||||
|
||||
|
||||
def _coco_images(
|
||||
value: object,
|
||||
*,
|
||||
expected: set[str],
|
||||
) -> dict[object, str]:
|
||||
result: dict[object, str] = {}
|
||||
observed: set[str] = set()
|
||||
for raw in _list(value, "COCO images"):
|
||||
row = _object(raw, "COCO image")
|
||||
image_id = row.get("id")
|
||||
name = Path(_string(row.get("file_name"), "COCO file name")).name
|
||||
if (
|
||||
isinstance(image_id, bool)
|
||||
or not isinstance(image_id, (int, str))
|
||||
or image_id in result
|
||||
or name in observed
|
||||
or row.get("width") != 800
|
||||
or row.get("height") != 600
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO image is invalid")
|
||||
result[image_id] = name
|
||||
observed.add(name)
|
||||
if observed != expected:
|
||||
raise E46ReviewHandoffError("COCO image coverage is incomplete")
|
||||
return result
|
||||
|
||||
|
||||
def _annotation_attributes(annotation: dict[str, Any]) -> dict[str, Any]:
|
||||
raw = annotation.get("attributes")
|
||||
if isinstance(raw, dict):
|
||||
result = dict(raw)
|
||||
elif isinstance(raw, list):
|
||||
result = {}
|
||||
for item in raw:
|
||||
row = _object(item, "COCO annotation attribute")
|
||||
name = row.get("name")
|
||||
if not isinstance(name, str) or name in result:
|
||||
raise E46ReviewHandoffError("COCO annotation attribute is invalid")
|
||||
result[name] = row.get("value")
|
||||
elif raw is None:
|
||||
result = {}
|
||||
else:
|
||||
raise E46ReviewHandoffError("COCO annotation attributes are invalid")
|
||||
for name in ("occluded", "truncated"):
|
||||
if name not in result and name in annotation:
|
||||
result[name] = annotation[name]
|
||||
return result
|
||||
|
||||
|
||||
def _required_bool(attributes: dict[str, Any], name: str) -> bool:
|
||||
value = attributes.get(name)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str) and value.lower() in {"true", "false"}:
|
||||
return value.lower() == "true"
|
||||
raise E46ReviewHandoffError(f"COCO annotation {name} flag is missing")
|
||||
|
||||
|
||||
def _coco_box(value: object) -> list[float]:
|
||||
raw = _list(value, "COCO bbox")
|
||||
if len(raw) != 4:
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
numbers: list[float] = []
|
||||
for item in raw:
|
||||
if isinstance(item, bool) or not isinstance(item, (int, float)):
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
number = float(item)
|
||||
if not math.isfinite(number):
|
||||
raise E46ReviewHandoffError("COCO bbox is invalid")
|
||||
numbers.append(number)
|
||||
x, y, width, height = numbers
|
||||
if (
|
||||
x < 0.0
|
||||
or y < 0.0
|
||||
or width <= 0.0
|
||||
or height <= 0.0
|
||||
or x + width > 800.0
|
||||
or y + height > 600.0
|
||||
):
|
||||
raise E46ReviewHandoffError("COCO bbox is outside the source image")
|
||||
return [x, y, x + width, y + height]
|
||||
|
||||
|
||||
def _safe_relative(root: Path, relative: str) -> Path:
|
||||
pure = PurePosixPath(relative)
|
||||
if pure.is_absolute() or ".." in pure.parts or not pure.parts:
|
||||
raise E46ReviewHandoffError("relative artifact path is unsafe")
|
||||
resolved = (root / Path(*pure.parts)).resolve(strict=True)
|
||||
try:
|
||||
resolved.relative_to(root.resolve(strict=True))
|
||||
except ValueError as reason:
|
||||
raise E46ReviewHandoffError(
|
||||
"relative artifact path escapes its root"
|
||||
) from reason
|
||||
return resolved
|
||||
|
||||
|
||||
def _write_zip(path: Path, entries: tuple[tuple[str, bytes], ...]) -> None:
|
||||
with zipfile.ZipFile(
|
||||
path,
|
||||
mode="w",
|
||||
compression=zipfile.ZIP_DEFLATED,
|
||||
compresslevel=6,
|
||||
) as archive:
|
||||
for name, payload in entries:
|
||||
info = zipfile.ZipInfo(name, date_time=_ZIP_TIMESTAMP)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o600 << 16
|
||||
archive.writestr(info, payload)
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_json_atomic(path: Path, value: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
||||
_write_json(temporary, value)
|
||||
os.replace(temporary, path)
|
||||
_fsync_directory(path.parent)
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
path.write_bytes(_canonical_json(value) + b"\n")
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.write_bytes(b"".join(_canonical_json(row) + b"\n" for row in rows))
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _write_text(path: Path, value: str) -> None:
|
||||
path.write_text(value, encoding="utf-8", newline="\n")
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
|
||||
def _artifact(path: Path, root: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"path": path.relative_to(root).as_posix(),
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(path.read_bytes())
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError(f"invalid JSON: {path.name}") from reason
|
||||
return _object(value, path.name)
|
||||
|
||||
|
||||
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
rows.append(_object(json.loads(line), path.name))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
|
||||
raise E46ReviewHandoffError(f"invalid JSONL: {path.name}") from reason
|
||||
return rows
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict):
|
||||
raise E46ReviewHandoffError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _list(value: object, label: str) -> list[Any]:
|
||||
if not isinstance(value, list):
|
||||
raise E46ReviewHandoffError(f"{label} must be a list")
|
||||
return value
|
||||
|
||||
|
||||
def _string(value: object, label: str) -> str:
|
||||
if not isinstance(value, str) or not value:
|
||||
raise E46ReviewHandoffError(f"{label} must be a string")
|
||||
return value
|
||||
|
||||
|
||||
def _positive_int(value: object, label: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise E46ReviewHandoffError(f"{label} must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, str) or len(value) > 1000:
|
||||
raise E46ReviewHandoffError("COCO annotation notes are invalid")
|
||||
return value
|
||||
|
||||
|
||||
def _utc_timestamp(value: object, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise E46ReviewHandoffError(f"{label} is invalid")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError as reason:
|
||||
raise E46ReviewHandoffError(f"{label} is invalid") from reason
|
||||
if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed):
|
||||
raise E46ReviewHandoffError(f"{label} must be UTC")
|
||||
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _fsync_tree(root: Path) -> None:
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_file():
|
||||
with path.open("rb") as stream:
|
||||
os.fsync(stream.fileno())
|
||||
elif path.is_dir():
|
||||
_fsync_directory(path)
|
||||
_fsync_directory(root)
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _handoff_readme(truth_island_id: str) -> str:
|
||||
return f"""# Mission Core E46 blind detector review
|
||||
|
||||
Truth island: `{truth_island_id}`
|
||||
|
||||
This handoff contains two equivalent reviewer slots. Give each slot to a
|
||||
different human reviewer. Reviewers must not see E47 predictions, scores,
|
||||
candidate identities, model prelabels or each other's annotations.
|
||||
|
||||
After both independent reviews are complete, convert each CVAT COCO export with
|
||||
`experiments/perception/convert_e46_cvat_review.py`. Adjudication and E48 sealing
|
||||
happen only after both converted submissions pass validation.
|
||||
"""
|
||||
|
||||
|
||||
def _slot_readme(slot: str) -> str:
|
||||
return f"""# E46 blind review slot: {slot}
|
||||
|
||||
1. Create a new CVAT image task with the seven labels in `labels.json`.
|
||||
2. Add boolean `occluded` and `truncated` attributes to every label.
|
||||
3. Upload `images.zip` as task data.
|
||||
4. Import `cvat-empty-coco.zip` as COCO 1.0 annotations. It intentionally
|
||||
contains zero objects and no model material.
|
||||
5. Draw every identifiable in-FOV instance. Frames with no objects remain empty.
|
||||
6. Set both attributes explicitly for every object.
|
||||
7. Export the completed task as COCO 1.0.
|
||||
|
||||
Do not use model assistance, prelabels, E47 predictions or another review.
|
||||
"""
|
||||
@@ -55,6 +55,36 @@ class E48DetectorTruthSealError(RuntimeError):
|
||||
"""An independent review or truth-seal artifact is invalid."""
|
||||
|
||||
|
||||
def validate_e48_detector_review_submission(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
review_path: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate one completed blind review before adjudication."""
|
||||
|
||||
try:
|
||||
truth_island = read_e46_detector_truth_island(truth_island_root)
|
||||
except E46DetectorTruthIslandError as reason:
|
||||
raise E48DetectorTruthSealError("E46 truth island is invalid") from reason
|
||||
references = tuple(
|
||||
_read_jsonl(truth_island.result_root / E46_REFERENCES_NAME)
|
||||
)
|
||||
contract = _read_json(truth_island.result_root / E46_CONTRACT_NAME)
|
||||
annotation = _object(contract.get("annotation"), "E46 annotation contract")
|
||||
raw_classes = annotation.get("classes")
|
||||
if not isinstance(raw_classes, list) or not all(
|
||||
isinstance(value, str) for value in raw_classes
|
||||
):
|
||||
raise E48DetectorTruthSealError("E46 target classes are invalid")
|
||||
document = _read_json(review_path.resolve(strict=True))
|
||||
return _validate_review(
|
||||
document,
|
||||
truth_island_id=truth_island.result_id,
|
||||
references=references,
|
||||
target_classes=frozenset(raw_classes),
|
||||
)
|
||||
|
||||
|
||||
def build_e48_detector_truth_seal(
|
||||
*,
|
||||
truth_island_root: Path,
|
||||
|
||||
Reference in New Issue
Block a user