feat(perception): prepare blind detector review handoff

This commit is contained in:
DCCONSTRUCTIONS
2026-07-30 00:03:41 +03:00
parent cc6838ed24
commit 091ab2671e
7 changed files with 1259 additions and 1 deletions
+1 -1
View File
@@ -15,7 +15,7 @@ Each gate produces evidence and an explicit GO, PAUSE or BLOCKED result.
| Evidence storage | GO (reference index only) — E50 converts the accepted E44 audit into 2,962 verified logical references over 1,029 canonical content objects, including 1,933 exact duplicate references and 212,717,913 measured duplicate bytes. Existing artifacts remain materialized and unchanged; physical reclamation and storage migration remain unauthorized. | | Evidence storage | GO (reference index only) — E50 converts the accepted E44 audit into 2,962 verified logical references over 1,029 canonical content objects, including 1,933 exact duplicate references and 212,717,913 measured duplicate bytes. Existing artifacts remain materialized and unchanged; physical reclamation and storage migration remain unauthorized. |
| Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. | | Future transfer | PREREGISTERED — E43 freezes same-K1/mount/calibration/firmware, required streams, connected-component split and independent label reveal. Capture and labels do not yet exist. |
| E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. | | E31 binding sensitivity | MEASURED — E45 closes accounting for 87/87 accepted correspondences and finds no material monotonic residual association with represented image radius, rig speed or pose age. It does not supply calibration-target truth or outer-fisheye coverage. |
| Detector Truth Island | PREPARED — E46 freezes 32 references with no prelabels, predictions, scores or candidate identity in the review package. Two independent reviews and adjudication are still required. | | Detector Truth Island | PREPARED FOR HUMAN REVIEW — E46 freezes 32 references with no prelabels, predictions, scores or candidate identity. Immutable handoff `e46-review-handoff-652f7409c8a529946a26264cea006f3afdd714e19529c731e0f66d0f03fd5668` provides two separately assigned, hash-verified CVAT/COCO reviewer packages and a fail-closed E48 converter. Two real independent reviews and adjudication are still required; the handoff is not ground truth. |
| Detector candidate comparison | FROZEN BEFORE TRUTH — E47 freezes raw-KB4 and fixed-valid-FOV-fill predictions from the same exact Mask R-CNN checkpoint. No accuracy result or winner exists before the E46 truth seal. | | Detector candidate comparison | FROZEN BEFORE TRUTH — E47 freezes raw-KB4 and fixed-valid-FOV-fill predictions from the same exact Mask R-CNN checkpoint. No accuracy result or winner exists before the E46 truth seal. |
| Detector truth/evaluation executors | READY, NOT RUN — E48 fail-closed review/adjudication sealing and separate post-seal E49 scoring are implemented. Neither result exists because real independent reviews are absent. | | Detector truth/evaluation executors | READY, NOT RUN — E48 fail-closed review/adjudication sealing and separate post-seal E49 scoring are implemented. Neither result exists because real independent reviews are absent. |
| Product interface | DEFERRED — no new windows, page anatomy or design changes are part of this stabilization increment. | | Product interface | DEFERRED — no new windows, page anatomy or design changes are part of this stabilization increment. |
@@ -168,6 +168,16 @@ prelabels, predictions, scores or candidate identity. It references
14,919,621 source bytes without copying images. The package is prepared, not 14,919,621 source bytes without copying images. The package is prepared, not
truth: two different human reviewers and adjudication remain mandatory. truth: two different human reviewers and adjudication remain mandatory.
Operational handoff
`e46-review-handoff-652f7409c8a529946a26264cea006f3afdd714e19529c731e0f66d0f03fd5668`
materializes those exact 32 hash-verified source frames into two separately
assignable CVAT/COCO slots. Both slots contain an empty annotation seed and the
same seven-class contract; neither contains predictions, prelabels, scores or
candidate identity. A converter validates exact frame coverage, allowed
classes, boxes and explicit occlusion/truncation fields before producing one
strict E48 review submission. This closes the technical handoff only: no human
review, adjudication or ground-truth seal is implied.
E47 immutable candidate freeze E47 immutable candidate freeze
`e47-detector-candidate-freeze-514bcca7a8a26313cab5ffcacca053d7a0ec6fe7cbef25f15faf3a11e48ee92f` `e47-detector-candidate-freeze-514bcca7a8a26313cab5ffcacca053d7a0ec6fe7cbef25f15faf3a11e48ee92f`
stores raw-KB4 and fixed-valid-FOV-fill predictions for those exact 32 frames. stores raw-KB4 and fixed-valid-FOV-fill predictions for those exact 32 frames.
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Convert one completed blind CVAT task into an E48 review submission."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e46_review_handoff import convert_cvat_coco_to_e48_review
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--handoff-root", type=Path, required=True)
parser.add_argument("--truth-island-root", type=Path, required=True)
parser.add_argument(
"--reviewer-slot",
choices=("reviewer-a", "reviewer-b"),
required=True,
)
parser.add_argument("--reviewer-id", required=True)
parser.add_argument("--cvat-export", type=Path, required=True)
parser.add_argument("--submitted-at-utc", required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
document = convert_cvat_coco_to_e48_review(
handoff_root=args.handoff_root,
truth_island_root=args.truth_island_root,
reviewer_slot=args.reviewer_slot,
reviewer_id=args.reviewer_id,
cvat_export_path=args.cvat_export,
submitted_at_utc=args.submitted_at_utc,
output_path=args.output,
)
print(
json.dumps(
{
"output": str(args.output.expanduser().absolute()),
"reviewer_id": document["reviewer_id"],
"frame_count": len(document["images"]),
"state": document["state"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Prepare two self-contained blind CVAT packages for E46."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from k1link.compute.e46_review_handoff import build_e46_review_handoff
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--truth-island-root", type=Path, required=True)
parser.add_argument("--evaluation-pack-root", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
args = parser.parse_args()
result = build_e46_review_handoff(
truth_island_root=args.truth_island_root,
evaluation_pack_root=args.evaluation_pack_root,
output_root=args.output_root,
)
print(
json.dumps(
{
"result_id": result.result_id,
"result_root": str(result.result_root),
"state": result.manifest["state"],
"frame_count": result.manifest["identity"]["frame_count"],
"reviewer_slots": result.manifest["identity"]["reviewer_slots"],
},
ensure_ascii=False,
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+839
View File
@@ -0,0 +1,839 @@
"""Operational blind-review handoff from E46 into human CVAT review."""
from __future__ import annotations
import hashlib
import json
import math
import os
import re
import shutil
import uuid
import zipfile
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Final
from PIL import Image
from .e46_detector_truth_island import (
E46_CONTRACT_NAME,
E46_MANIFEST_NAME,
E46_REFERENCES_NAME,
E46_REVIEW_NAME,
E46DetectorTruthIslandError,
read_e46_detector_truth_island,
)
from .e48_detector_truth_seal import (
E48_REVIEW_SCHEMA,
E48DetectorTruthSealError,
validate_e48_detector_review_submission,
)
E46_REVIEW_HANDOFF_SCHEMA: Final = "missioncore.e46-review-handoff/v1"
E46_REVIEW_HANDOFF_PROFILE: Final = "cvat-coco-empty-blind-two-reviewer/v1"
E46_REVIEW_SOURCE_SCHEMA: Final = "missioncore.e46-review-source/v1"
E46_REVIEW_HANDOFF_MANIFEST: Final = "manifest.json"
E46_REVIEW_HANDOFF_SLOTS: Final = ("reviewer-a", "reviewer-b")
_AUTHORITY: Final = {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
}
_BLINDNESS: Final = {
"candidate_identity_seen": False,
"model_prelabels_seen": False,
"model_predictions_seen": False,
"model_scores_seen": False,
}
_SAFE_HANDOFF = re.compile(r"^e46-review-handoff-[a-f0-9]{64}$")
_SAFE_REVIEWER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,63}$")
_ZIP_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0)
class E46ReviewHandoffError(RuntimeError):
"""The E46 reviewer handoff or completed CVAT export is invalid."""
@dataclass(frozen=True, slots=True)
class E46ReviewHandoff:
result_id: str
result_root: Path
manifest: dict[str, Any]
def build_e46_review_handoff(
*,
truth_island_root: Path,
evaluation_pack_root: Path,
output_root: Path,
) -> E46ReviewHandoff:
"""Create two self-contained, prediction-free reviewer packages."""
try:
truth = read_e46_detector_truth_island(truth_island_root)
except E46DetectorTruthIslandError as reason:
raise E46ReviewHandoffError("E46 truth island is invalid") from reason
evaluation_root = evaluation_pack_root.resolve(strict=True)
evaluation_manifest = _read_json(evaluation_root / "manifest.json")
truth_source = _object(
_object(truth.manifest.get("identity"), "E46 identity").get("source"),
"E46 source",
)
if (
evaluation_manifest.get("generation_id") != evaluation_root.name
or truth_source.get("evaluation_pack_id") != evaluation_root.name
):
raise E46ReviewHandoffError("E2 evaluation pack identity changed")
references = tuple(_read_jsonl(truth.result_root / E46_REFERENCES_NAME))
contract = _read_json(truth.result_root / E46_CONTRACT_NAME)
review_template = _read_json(truth.result_root / E46_REVIEW_NAME)
annotation = _object(contract.get("annotation"), "E46 annotation contract")
raw_classes = annotation.get("classes")
if (
contract.get("reviewer_package", {}).get("model_predictions_included")
is not False
or contract.get("reviewer_package", {}).get("model_prelabels_included")
is not False
or not isinstance(raw_classes, list)
or not raw_classes
or not all(isinstance(value, str) and value for value in raw_classes)
):
raise E46ReviewHandoffError("E46 blind annotation contract is invalid")
classes = tuple(raw_classes)
if (
review_template.get("truth_island_id") != truth.result_id
or len(_list(review_template.get("images"), "E46 review images"))
!= len(references)
):
raise E46ReviewHandoffError("E46 review template is invalid")
sources = tuple(
_verified_source(
evaluation_root=evaluation_root,
reference=reference,
)
for reference in references
)
identity = {
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
"profile": E46_REVIEW_HANDOFF_PROFILE,
"truth_island": {
"result_id": truth.result_id,
"manifest_sha256": _sha256(truth.result_root / E46_MANIFEST_NAME),
"references_sha256": _sha256(
truth.result_root / E46_REFERENCES_NAME
),
},
"evaluation_pack": {
"generation_id": evaluation_root.name,
"manifest_sha256": _sha256(evaluation_root / "manifest.json"),
},
"reviewer_slots": list(E46_REVIEW_HANDOFF_SLOTS),
"frame_count": len(sources),
"classes": list(classes),
"blindness": _BLINDNESS,
"authority": _AUTHORITY,
}
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
result_id = f"e46-review-handoff-{identity_sha256}"
parent = output_root.expanduser().absolute()
destination = parent / result_id
if destination.exists():
return validate_e46_review_handoff(destination)
parent.mkdir(mode=0o700, parents=True, exist_ok=True)
staging = parent / f".{result_id}.{uuid.uuid4().hex}.tmp"
staging.mkdir(mode=0o700)
try:
for slot in E46_REVIEW_HANDOFF_SLOTS:
_write_reviewer_slot(
root=staging / slot,
slot=slot,
truth_island_id=truth.result_id,
sources=sources,
classes=classes,
review_template=review_template,
)
_write_text(staging / "README.md", _handoff_readme(truth.result_id))
artifacts = [
_artifact(path, staging)
for path in sorted(staging.rglob("*"))
if path.is_file()
]
manifest = {
"schema_version": E46_REVIEW_HANDOFF_SCHEMA,
"result_id": result_id,
"identity_sha256": identity_sha256,
"identity": identity,
"created_at_utc": _utc_now(),
"state": "prepared-awaiting-two-independent-human-reviews",
"ground_truth": False,
"artifacts": artifacts,
"authority": _AUTHORITY,
}
_write_json(staging / E46_REVIEW_HANDOFF_MANIFEST, manifest)
_fsync_tree(staging)
os.replace(staging, destination)
_fsync_directory(parent)
except BaseException:
shutil.rmtree(staging, ignore_errors=True)
raise
return validate_e46_review_handoff(destination)
def validate_e46_review_handoff(root: Path) -> E46ReviewHandoff:
"""Revalidate a complete E46 blind-review handoff."""
resolved = root.resolve(strict=True)
manifest = _read_json(resolved / E46_REVIEW_HANDOFF_MANIFEST)
identity = _object(manifest.get("identity"), "handoff identity")
identity_sha256 = manifest.get("identity_sha256")
if (
not resolved.is_dir()
or _SAFE_HANDOFF.fullmatch(resolved.name) is None
or manifest.get("schema_version") != E46_REVIEW_HANDOFF_SCHEMA
or manifest.get("result_id") != resolved.name
or manifest.get("state")
!= "prepared-awaiting-two-independent-human-reviews"
or manifest.get("ground_truth") is not False
or manifest.get("authority") != _AUTHORITY
or not isinstance(identity_sha256, str)
or hashlib.sha256(_canonical_json(identity)).hexdigest()
!= identity_sha256
or resolved.name != f"e46-review-handoff-{identity_sha256}"
or identity.get("profile") != E46_REVIEW_HANDOFF_PROFILE
or identity.get("blindness") != _BLINDNESS
or identity.get("reviewer_slots") != list(E46_REVIEW_HANDOFF_SLOTS)
):
raise E46ReviewHandoffError("E46 review handoff identity is invalid")
artifacts = _list(manifest.get("artifacts"), "handoff artifacts")
artifact_paths: set[str] = set()
for raw in artifacts:
artifact = _object(raw, "handoff artifact")
relative = artifact.get("path")
if not isinstance(relative, str):
raise E46ReviewHandoffError("handoff artifact path is invalid")
path = _safe_relative(resolved, relative)
if (
not path.is_file()
or path.is_symlink()
or path.stat().st_size != artifact.get("byte_length")
or _sha256(path) != artifact.get("sha256")
):
raise E46ReviewHandoffError("handoff artifact changed")
artifact_paths.add(relative)
required = {"README.md"}
for slot in E46_REVIEW_HANDOFF_SLOTS:
required.update(
{
f"{slot}/README.md",
f"{slot}/images.zip",
f"{slot}/cvat-empty-coco.zip",
f"{slot}/labels.json",
f"{slot}/source-map.jsonl",
f"{slot}/submission-template.json",
}
)
if not required.issubset(artifact_paths):
raise E46ReviewHandoffError("reviewer handoff artifacts are incomplete")
for slot in E46_REVIEW_HANDOFF_SLOTS:
_validate_slot(
resolved / slot,
expected_frame_count=_positive_int(
identity.get("frame_count"),
"handoff frame count",
),
expected_classes=tuple(
_string(value, "handoff class")
for value in _list(identity.get("classes"), "handoff classes")
),
)
return E46ReviewHandoff(
result_id=resolved.name,
result_root=resolved,
manifest=manifest,
)
def convert_cvat_coco_to_e48_review(
*,
handoff_root: Path,
truth_island_root: Path,
reviewer_slot: str,
reviewer_id: str,
cvat_export_path: Path,
submitted_at_utc: str,
output_path: Path,
) -> dict[str, Any]:
"""Convert one completed CVAT COCO export into a strict E48 review."""
handoff = validate_e46_review_handoff(handoff_root)
if reviewer_slot not in E46_REVIEW_HANDOFF_SLOTS:
raise E46ReviewHandoffError("reviewer slot is invalid")
if _SAFE_REVIEWER.fullmatch(reviewer_id) is None:
raise E46ReviewHandoffError("reviewer identity is invalid")
submitted = _utc_timestamp(submitted_at_utc, "review submitted_at_utc")
slot_root = handoff.result_root / reviewer_slot
source_rows = tuple(_read_jsonl(slot_root / "source-map.jsonl"))
template = _read_json(slot_root / "submission-template.json")
coco = _read_cvat_coco(cvat_export_path)
categories = _coco_categories(
coco.get("categories"),
allowed=frozenset(handoff.manifest["identity"]["classes"]),
)
image_names = _coco_images(
coco.get("images"),
expected={str(row["file_name"]) for row in source_rows},
)
objects_by_name: dict[str, list[dict[str, Any]]] = {
str(row["file_name"]): [] for row in source_rows
}
raw_annotations = _list(coco.get("annotations"), "COCO annotations")
for raw_annotation in raw_annotations:
annotation = _object(raw_annotation, "COCO annotation")
image_id = annotation.get("image_id")
category_id = annotation.get("category_id")
if image_id not in image_names or category_id not in categories:
raise E46ReviewHandoffError("COCO annotation identity is invalid")
attributes = _annotation_attributes(annotation)
objects_by_name[image_names[image_id]].append(
{
"category": categories[category_id],
"box_xyxy": _coco_box(annotation.get("bbox")),
"occluded": _required_bool(attributes, "occluded"),
"truncated": _required_bool(attributes, "truncated"),
"notes": _optional_text(attributes.get("notes")),
}
)
images: list[dict[str, Any]] = []
template_images = _list(template.get("images"), "submission images")
if len(template_images) != len(source_rows):
raise E46ReviewHandoffError("submission template coverage changed")
for raw_template, source in zip(template_images, source_rows, strict=True):
image = dict(_object(raw_template, "submission image"))
file_name = str(source["file_name"])
raw_objects = sorted(
objects_by_name[file_name],
key=lambda item: (
str(item["category"]),
tuple(float(value) for value in item["box_xyxy"]),
),
)
image["review_state"] = "reviewed"
image["objects"] = [
{"object_id": f"object-{index:03d}", **item}
for index, item in enumerate(raw_objects, start=1)
]
image["hard_negative"] = not raw_objects
image["notes"] = None
images.append(image)
document = {
"schema_version": E48_REVIEW_SCHEMA,
"truth_island_id": template["truth_island_id"],
"state": "completed-independent-no-model-assistance",
"reviewer_id": reviewer_id,
"review_round": 1,
"blindness": _BLINDNESS,
"images": images,
"acceptance": {
"all_images_reviewed": True,
"independent": True,
"submitted_at_utc": submitted,
},
}
_write_json_atomic(output_path, document)
try:
validate_e48_detector_review_submission(
truth_island_root=truth_island_root,
review_path=output_path,
)
except E48DetectorTruthSealError as reason:
raise E46ReviewHandoffError(
"converted E48 review is invalid"
) from reason
return document
def _write_reviewer_slot(
*,
root: Path,
slot: str,
truth_island_id: str,
sources: tuple[dict[str, Any], ...],
classes: tuple[str, ...],
review_template: dict[str, Any],
) -> None:
root.mkdir(mode=0o700, parents=True)
file_payloads = tuple(
(str(source["file_name"]), Path(source["path"]).read_bytes())
for source in sources
)
_write_zip(root / "images.zip", file_payloads)
coco = {
"info": {
"description": "Mission Core E46 blind detector review",
"version": E46_REVIEW_HANDOFF_PROFILE,
"truth_island_id": truth_island_id,
"reviewer_slot": slot,
"model_material_included": False,
},
"licenses": [],
"images": [
{
"id": int(source["truth_island_sequence"]),
"width": 800,
"height": 600,
"file_name": source["file_name"],
}
for source in sources
],
"annotations": [],
"categories": [
{"id": index, "name": name, "supercategory": "missioncore"}
for index, name in enumerate(classes, start=1)
],
}
_write_zip(
root / "cvat-empty-coco.zip",
(("annotations/instances_default.json", _canonical_json(coco) + b"\n"),),
)
labels = {
"schema_version": "missioncore.e46-cvat-labels/v1",
"labels": [
{
"name": name,
"attributes": [
{
"name": "occluded",
"input_type": "checkbox",
"default_value": "false",
},
{
"name": "truncated",
"input_type": "checkbox",
"default_value": "false",
},
],
}
for name in classes
],
}
_write_json(root / "labels.json", labels)
source_rows = [
{
"schema_version": E46_REVIEW_SOURCE_SCHEMA,
"truth_island_sequence": source["truth_island_sequence"],
"file_name": source["file_name"],
"source_path": source["source_path"],
"source_sha256": source["source_sha256"],
"byte_length": source["byte_length"],
}
for source in sources
]
_write_jsonl(root / "source-map.jsonl", source_rows)
submission = json.loads(json.dumps(review_template))
submission["schema_version"] = E48_REVIEW_SCHEMA
submission["state"] = "prepared-awaiting-independent-review"
submission["reviewer_id"] = None
submission["review_round"] = 1
submission["blindness"] = _BLINDNESS
submission["acceptance"] = None
_write_json(root / "submission-template.json", submission)
_write_text(root / "README.md", _slot_readme(slot))
def _verified_source(
*,
evaluation_root: Path,
reference: dict[str, Any],
) -> dict[str, Any]:
relative = _string(reference.get("source_path"), "E46 source path")
path = _safe_relative(evaluation_root, relative)
if path.is_symlink() or not path.is_file():
raise E46ReviewHandoffError("E46 source image is unavailable")
byte_length = _positive_int(reference.get("byte_length"), "source bytes")
source_sha256 = _string(reference.get("sha256"), "source SHA-256")
if path.stat().st_size != byte_length or _sha256(path) != source_sha256:
raise E46ReviewHandoffError("E46 source image identity changed")
with Image.open(path) as image:
if image.format != "PNG" or image.size != (800, 600):
raise E46ReviewHandoffError("E46 source image geometry changed")
image.verify()
sequence = _positive_int(
reference.get("truth_island_sequence"),
"truth island sequence",
)
return {
"truth_island_sequence": sequence,
"file_name": f"e46-{sequence:04d}-{Path(relative).name}",
"source_path": relative,
"source_sha256": source_sha256,
"byte_length": byte_length,
"path": path,
}
def _validate_slot(
root: Path,
*,
expected_frame_count: int,
expected_classes: tuple[str, ...],
) -> None:
source_rows = tuple(_read_jsonl(root / "source-map.jsonl"))
if (
len(source_rows) != expected_frame_count
or len({row.get("file_name") for row in source_rows})
!= expected_frame_count
):
raise E46ReviewHandoffError("reviewer source map is invalid")
with zipfile.ZipFile(root / "images.zip") as archive:
names = archive.namelist()
if names != [row["file_name"] for row in source_rows]:
raise E46ReviewHandoffError("reviewer image archive order changed")
for row in source_rows:
payload = archive.read(str(row["file_name"]))
if (
len(payload) != row.get("byte_length")
or hashlib.sha256(payload).hexdigest()
!= row.get("source_sha256")
):
raise E46ReviewHandoffError("reviewer image payload changed")
coco = _read_zip_json(root / "cvat-empty-coco.zip")
if (
coco.get("annotations") != []
or len(_list(coco.get("images"), "handoff COCO images"))
!= expected_frame_count
or tuple(
_string(row.get("name"), "handoff COCO category")
for row in _list(coco.get("categories"), "handoff COCO categories")
)
!= expected_classes
):
raise E46ReviewHandoffError("blind CVAT payload changed")
def _read_cvat_coco(path: Path) -> dict[str, Any]:
resolved = path.resolve(strict=True)
if resolved.suffix.lower() == ".zip":
return _read_zip_json(resolved)
return _read_json(resolved)
def _read_zip_json(path: Path) -> dict[str, Any]:
with zipfile.ZipFile(path) as archive:
json_names = [
name
for name in archive.namelist()
if PurePosixPath(name).suffix.lower() == ".json"
]
if len(json_names) != 1:
raise E46ReviewHandoffError(
"CVAT COCO archive must contain one JSON document"
)
try:
value = json.loads(archive.read(json_names[0]))
except (json.JSONDecodeError, UnicodeDecodeError) as reason:
raise E46ReviewHandoffError("CVAT COCO JSON is invalid") from reason
return _object(value, "CVAT COCO document")
def _coco_categories(
value: object,
*,
allowed: frozenset[str],
) -> dict[object, str]:
result: dict[object, str] = {}
for raw in _list(value, "COCO categories"):
row = _object(raw, "COCO category")
category_id = row.get("id")
name = row.get("name")
if (
isinstance(category_id, bool)
or not isinstance(category_id, (int, str))
or not isinstance(name, str)
or name not in allowed
or category_id in result
):
raise E46ReviewHandoffError("COCO category is invalid")
result[category_id] = name
if not result:
raise E46ReviewHandoffError("COCO categories are empty")
return result
def _coco_images(
value: object,
*,
expected: set[str],
) -> dict[object, str]:
result: dict[object, str] = {}
observed: set[str] = set()
for raw in _list(value, "COCO images"):
row = _object(raw, "COCO image")
image_id = row.get("id")
name = Path(_string(row.get("file_name"), "COCO file name")).name
if (
isinstance(image_id, bool)
or not isinstance(image_id, (int, str))
or image_id in result
or name in observed
or row.get("width") != 800
or row.get("height") != 600
):
raise E46ReviewHandoffError("COCO image is invalid")
result[image_id] = name
observed.add(name)
if observed != expected:
raise E46ReviewHandoffError("COCO image coverage is incomplete")
return result
def _annotation_attributes(annotation: dict[str, Any]) -> dict[str, Any]:
raw = annotation.get("attributes")
if isinstance(raw, dict):
result = dict(raw)
elif isinstance(raw, list):
result = {}
for item in raw:
row = _object(item, "COCO annotation attribute")
name = row.get("name")
if not isinstance(name, str) or name in result:
raise E46ReviewHandoffError("COCO annotation attribute is invalid")
result[name] = row.get("value")
elif raw is None:
result = {}
else:
raise E46ReviewHandoffError("COCO annotation attributes are invalid")
for name in ("occluded", "truncated"):
if name not in result and name in annotation:
result[name] = annotation[name]
return result
def _required_bool(attributes: dict[str, Any], name: str) -> bool:
value = attributes.get(name)
if isinstance(value, bool):
return value
if isinstance(value, str) and value.lower() in {"true", "false"}:
return value.lower() == "true"
raise E46ReviewHandoffError(f"COCO annotation {name} flag is missing")
def _coco_box(value: object) -> list[float]:
raw = _list(value, "COCO bbox")
if len(raw) != 4:
raise E46ReviewHandoffError("COCO bbox is invalid")
numbers: list[float] = []
for item in raw:
if isinstance(item, bool) or not isinstance(item, (int, float)):
raise E46ReviewHandoffError("COCO bbox is invalid")
number = float(item)
if not math.isfinite(number):
raise E46ReviewHandoffError("COCO bbox is invalid")
numbers.append(number)
x, y, width, height = numbers
if (
x < 0.0
or y < 0.0
or width <= 0.0
or height <= 0.0
or x + width > 800.0
or y + height > 600.0
):
raise E46ReviewHandoffError("COCO bbox is outside the source image")
return [x, y, x + width, y + height]
def _safe_relative(root: Path, relative: str) -> Path:
pure = PurePosixPath(relative)
if pure.is_absolute() or ".." in pure.parts or not pure.parts:
raise E46ReviewHandoffError("relative artifact path is unsafe")
resolved = (root / Path(*pure.parts)).resolve(strict=True)
try:
resolved.relative_to(root.resolve(strict=True))
except ValueError as reason:
raise E46ReviewHandoffError(
"relative artifact path escapes its root"
) from reason
return resolved
def _write_zip(path: Path, entries: tuple[tuple[str, bytes], ...]) -> None:
with zipfile.ZipFile(
path,
mode="w",
compression=zipfile.ZIP_DEFLATED,
compresslevel=6,
) as archive:
for name, payload in entries:
info = zipfile.ZipInfo(name, date_time=_ZIP_TIMESTAMP)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o600 << 16
archive.writestr(info, payload)
os.chmod(path, 0o600)
def _write_json_atomic(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
temporary = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
_write_json(temporary, value)
os.replace(temporary, path)
_fsync_directory(path.parent)
def _write_json(path: Path, value: object) -> None:
path.write_bytes(_canonical_json(value) + b"\n")
os.chmod(path, 0o600)
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
path.write_bytes(b"".join(_canonical_json(row) + b"\n" for row in rows))
os.chmod(path, 0o600)
def _write_text(path: Path, value: str) -> None:
path.write_text(value, encoding="utf-8", newline="\n")
os.chmod(path, 0o600)
def _artifact(path: Path, root: Path) -> dict[str, Any]:
return {
"path": path.relative_to(root).as_posix(),
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _read_json(path: Path) -> dict[str, Any]:
try:
value = json.loads(path.read_bytes())
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
raise E46ReviewHandoffError(f"invalid JSON: {path.name}") from reason
return _object(value, path.name)
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
try:
for line in path.read_text(encoding="utf-8").splitlines():
rows.append(_object(json.loads(line), path.name))
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as reason:
raise E46ReviewHandoffError(f"invalid JSONL: {path.name}") from reason
return rows
def _canonical_json(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _object(value: object, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise E46ReviewHandoffError(f"{label} must be an object")
return value
def _list(value: object, label: str) -> list[Any]:
if not isinstance(value, list):
raise E46ReviewHandoffError(f"{label} must be a list")
return value
def _string(value: object, label: str) -> str:
if not isinstance(value, str) or not value:
raise E46ReviewHandoffError(f"{label} must be a string")
return value
def _positive_int(value: object, label: str) -> int:
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise E46ReviewHandoffError(f"{label} must be a positive integer")
return value
def _optional_text(value: object) -> str | None:
if value is None:
return None
if not isinstance(value, str) or len(value) > 1000:
raise E46ReviewHandoffError("COCO annotation notes are invalid")
return value
def _utc_timestamp(value: object, label: str) -> str:
if not isinstance(value, str):
raise E46ReviewHandoffError(f"{label} is invalid")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as reason:
raise E46ReviewHandoffError(f"{label} is invalid") from reason
if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed):
raise E46ReviewHandoffError(f"{label} must be UTC")
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
def _utc_now() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
def _fsync_tree(root: Path) -> None:
for path in sorted(root.rglob("*"), reverse=True):
if path.is_file():
with path.open("rb") as stream:
os.fsync(stream.fileno())
elif path.is_dir():
_fsync_directory(path)
_fsync_directory(root)
def _fsync_directory(path: Path) -> None:
descriptor = os.open(path, os.O_RDONLY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def _handoff_readme(truth_island_id: str) -> str:
return f"""# Mission Core E46 blind detector review
Truth island: `{truth_island_id}`
This handoff contains two equivalent reviewer slots. Give each slot to a
different human reviewer. Reviewers must not see E47 predictions, scores,
candidate identities, model prelabels or each other's annotations.
After both independent reviews are complete, convert each CVAT COCO export with
`experiments/perception/convert_e46_cvat_review.py`. Adjudication and E48 sealing
happen only after both converted submissions pass validation.
"""
def _slot_readme(slot: str) -> str:
return f"""# E46 blind review slot: {slot}
1. Create a new CVAT image task with the seven labels in `labels.json`.
2. Add boolean `occluded` and `truncated` attributes to every label.
3. Upload `images.zip` as task data.
4. Import `cvat-empty-coco.zip` as COCO 1.0 annotations. It intentionally
contains zero objects and no model material.
5. Draw every identifiable in-FOV instance. Frames with no objects remain empty.
6. Set both attributes explicitly for every object.
7. Export the completed task as COCO 1.0.
Do not use model assistance, prelabels, E47 predictions or another review.
"""
@@ -55,6 +55,36 @@ class E48DetectorTruthSealError(RuntimeError):
"""An independent review or truth-seal artifact is invalid.""" """An independent review or truth-seal artifact is invalid."""
def validate_e48_detector_review_submission(
*,
truth_island_root: Path,
review_path: Path,
) -> dict[str, Any]:
"""Validate one completed blind review before adjudication."""
try:
truth_island = read_e46_detector_truth_island(truth_island_root)
except E46DetectorTruthIslandError as reason:
raise E48DetectorTruthSealError("E46 truth island is invalid") from reason
references = tuple(
_read_jsonl(truth_island.result_root / E46_REFERENCES_NAME)
)
contract = _read_json(truth_island.result_root / E46_CONTRACT_NAME)
annotation = _object(contract.get("annotation"), "E46 annotation contract")
raw_classes = annotation.get("classes")
if not isinstance(raw_classes, list) or not all(
isinstance(value, str) for value in raw_classes
):
raise E48DetectorTruthSealError("E46 target classes are invalid")
document = _read_json(review_path.resolve(strict=True))
return _validate_review(
document,
truth_island_id=truth_island.result_id,
references=references,
target_classes=frozenset(raw_classes),
)
def build_e48_detector_truth_seal( def build_e48_detector_truth_seal(
*, *,
truth_island_root: Path, truth_island_root: Path,
+286
View File
@@ -0,0 +1,286 @@
from __future__ import annotations
import hashlib
import json
import zipfile
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
from PIL import Image
from k1link.compute import e46_review_handoff as handoff
from k1link.compute import e48_detector_truth_seal as e48
def _canonical(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode()
def _write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(_canonical(value) + b"\n")
def _fixture(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> tuple[Path, Path, SimpleNamespace]:
truth_root = tmp_path / ("e46-detector-truth-island-" + "a" * 64)
evaluation_root = tmp_path / ("evaluation-pack-" + "b" * 64)
truth_root.mkdir()
image_root = evaluation_root / "images" / "valid-fov-fill"
image_root.mkdir(parents=True)
references: list[dict[str, Any]] = []
review_images: list[dict[str, Any]] = []
for sequence, image_id in enumerate((2, 5), start=1):
frame_index = sequence * 10
name = f"image-{image_id:03d}-frame-{frame_index:06d}.png"
path = image_root / name
Image.new("RGB", (800, 600), (sequence * 30, 10, 20)).save(path)
digest = hashlib.sha256(path.read_bytes()).hexdigest()
relative = f"images/valid-fov-fill/{name}"
reference = {
"schema_version": "missioncore.e46-truth-island-image-reference/v1",
"truth_island_sequence": sequence,
"image_id": image_id,
"frame_index": frame_index,
"session_seconds": float(sequence),
"role": "anchor",
"group_id": f"anchor-{sequence}",
"source_path": relative,
"sha256": digest,
"byte_length": path.stat().st_size,
}
references.append(reference)
review_images.append(
{
"truth_island_sequence": sequence,
"image_id": image_id,
"frame_index": frame_index,
"session_seconds": float(sequence),
"role": "anchor",
"group_id": f"anchor-{sequence}",
"source_path": relative,
"source_sha256": digest,
"review_state": "pending",
"hard_negative": None,
"objects": [],
"notes": None,
}
)
_write_json(
evaluation_root / "manifest.json",
{"generation_id": evaluation_root.name},
)
(truth_root / "image-references.jsonl").write_bytes(
b"".join(_canonical(row) + b"\n" for row in references)
)
_write_json(
truth_root / "blind-contract.json",
{
"annotation": {"classes": ["person", "car"]},
"reviewer_package": {
"model_predictions_included": False,
"model_prelabels_included": False,
},
},
)
_write_json(
truth_root / "review-template.json",
{
"schema_version": "missioncore.e46-detector-review-template/v1",
"truth_island_id": truth_root.name,
"state": "prepared-unreviewed-no-prelabels",
"reviewer_id": None,
"review_round": None,
"images": review_images,
"acceptance": None,
},
)
manifest = {
"identity": {
"source": {"evaluation_pack_id": evaluation_root.name},
}
}
_write_json(truth_root / "manifest.json", manifest)
truth = SimpleNamespace(
result_id=truth_root.name,
result_root=truth_root,
manifest=manifest,
report={"status": "prepared-awaiting-independent-human-review"},
)
monkeypatch.setattr(
handoff,
"read_e46_detector_truth_island",
lambda _root: truth,
)
monkeypatch.setattr(
e48,
"read_e46_detector_truth_island",
lambda _root: truth,
)
return truth_root, evaluation_root, truth
def test_handoff_is_blind_reproducible_and_converts_cvat(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
result = handoff.build_e46_review_handoff(
truth_island_root=truth_root,
evaluation_pack_root=evaluation_root,
output_root=tmp_path / "handoffs",
)
repeated = handoff.build_e46_review_handoff(
truth_island_root=truth_root,
evaluation_pack_root=evaluation_root,
output_root=tmp_path / "handoffs",
)
assert repeated.result_id == result.result_id
assert result.manifest["identity"]["frame_count"] == 2
with zipfile.ZipFile(
result.result_root / "reviewer-a" / "cvat-empty-coco.zip"
) as archive:
coco = json.loads(archive.read("annotations/instances_default.json"))
assert coco["annotations"] == []
assert "predictions" not in json.dumps(coco).lower()
source_rows = [
json.loads(line)
for line in (
result.result_root / "reviewer-a" / "source-map.jsonl"
).read_text().splitlines()
]
cvat = {
"images": [
{
"id": index,
"file_name": row["file_name"],
"width": 800,
"height": 600,
}
for index, row in enumerate(source_rows, start=11)
],
"categories": [
{"id": 1, "name": "person"},
{"id": 2, "name": "car"},
],
"annotations": [
{
"id": 1,
"image_id": 11,
"category_id": 2,
"bbox": [10.0, 20.0, 100.0, 200.0],
"attributes": {
"occluded": False,
"truncated": True,
},
}
],
}
cvat_path = tmp_path / "review-a.json"
_write_json(cvat_path, cvat)
output_path = tmp_path / "e48-review-a.json"
document = handoff.convert_cvat_coco_to_e48_review(
handoff_root=result.result_root,
truth_island_root=truth_root,
reviewer_slot="reviewer-a",
reviewer_id="human-reviewer-a",
cvat_export_path=cvat_path,
submitted_at_utc="2026-07-29T20:00:00Z",
output_path=output_path,
)
assert document["state"] == "completed-independent-no-model-assistance"
assert document["images"][0]["objects"][0]["box_xyxy"] == [
10.0,
20.0,
110.0,
220.0,
]
assert document["images"][1]["hard_negative"] is True
assert e48.validate_e48_detector_review_submission(
truth_island_root=truth_root,
review_path=output_path,
)["reviewer_id"] == "human-reviewer-a"
def test_handoff_rejects_changed_source_image(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
source = next(
(evaluation_root / "images" / "valid-fov-fill").glob("*.png")
)
source.write_bytes(b"changed")
with pytest.raises(handoff.E46ReviewHandoffError, match="identity changed"):
handoff.build_e46_review_handoff(
truth_island_root=truth_root,
evaluation_pack_root=evaluation_root,
output_root=tmp_path / "handoffs",
)
def test_converter_requires_explicit_occlusion_and_truncation(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
truth_root, evaluation_root, _truth = _fixture(tmp_path, monkeypatch)
result = handoff.build_e46_review_handoff(
truth_island_root=truth_root,
evaluation_pack_root=evaluation_root,
output_root=tmp_path / "handoffs",
)
source_rows = [
json.loads(line)
for line in (
result.result_root / "reviewer-a" / "source-map.jsonl"
).read_text().splitlines()
]
cvat = {
"images": [
{
"id": index,
"file_name": row["file_name"],
"width": 800,
"height": 600,
}
for index, row in enumerate(source_rows, start=1)
],
"categories": [{"id": 1, "name": "car"}],
"annotations": [
{
"id": 1,
"image_id": 1,
"category_id": 1,
"bbox": [10.0, 20.0, 100.0, 200.0],
}
],
}
cvat_path = tmp_path / "invalid.json"
_write_json(cvat_path, cvat)
with pytest.raises(
handoff.E46ReviewHandoffError,
match="occluded flag is missing",
):
handoff.convert_cvat_coco_to_e48_review(
handoff_root=result.result_root,
truth_island_root=truth_root,
reviewer_slot="reviewer-a",
reviewer_id="human-reviewer-a",
cvat_export_path=cvat_path,
submitted_at_utc="2026-07-29T20:00:00Z",
output_path=tmp_path / "invalid-review.json",
)