DCCONSTRUCTIONS
b53d6d5a45
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
778 lines
32 KiB
Python
778 lines
32 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from PIL import Image
|
|
|
|
from k1link.artifacts import utc_now_iso, write_json_atomic
|
|
from k1link.device_plugins.xgrids_k1.analyze.valid_fov import (
|
|
validate_k1_valid_fov_mask,
|
|
)
|
|
from k1link.sessions import SessionIntegrityError
|
|
|
|
from .jobs import validate_camera_compute_job
|
|
from .qualification import validate_recorded_qualification_slice
|
|
|
|
EVALUATION_PACK_SCHEMA = "missioncore.perception-evaluation-pack/v1"
|
|
EVALUATION_PACK_IDENTITY_SCHEMA = "missioncore.perception-evaluation-pack-identity/v1"
|
|
ANNOTATION_CONTRACT_SCHEMA = "missioncore.perception-annotation-contract/v1"
|
|
ANNOTATION_TEMPLATE_SCHEMA = "missioncore.perception-annotation-template/v1"
|
|
SELECTION_POLICY = "reviewed-anchors-plus-temporal-clips/v1"
|
|
MIN_EVALUATION_FRAMES = 16
|
|
MAX_EVALUATION_FRAMES = 128
|
|
MAX_MANIFEST_BYTES = 32 * 1024 * 1024
|
|
MAX_TIMELINE_LINE_BYTES = 16 * 1024
|
|
|
|
_SHA256 = re.compile(r"^[a-f0-9]{64}$")
|
|
_SAFE_GENERATION = re.compile(r"^evaluation-pack-[a-f0-9]{64}$")
|
|
_SAFE_GROUP = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
|
|
|
|
class RecordedEvaluationPackError(RuntimeError):
|
|
"""A recorded-perception evaluation pack is invalid."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class EvaluationFrameRequest:
|
|
frame_index: int
|
|
role: str
|
|
group_id: str
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class EvaluationPackFrame:
|
|
image_id: int
|
|
frame_index: int
|
|
sequence: int
|
|
segment_sha256: str
|
|
session_seconds: float
|
|
role: str
|
|
group_id: str
|
|
raw_path: Path
|
|
valid_fov_fill_path: Path
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class RecordedEvaluationPack:
|
|
generation_id: str
|
|
root: Path
|
|
manifest_path: Path
|
|
annotation_contract_path: Path
|
|
annotation_template_path: Path
|
|
job_id: str
|
|
input_sha256: str
|
|
qualification_generation_id: str
|
|
valid_fov_generation_id: str
|
|
calibration_sha256: str
|
|
frames: tuple[EvaluationPackFrame, ...]
|
|
|
|
|
|
def prepare_recorded_evaluation_pack(
|
|
*,
|
|
job_root: Path,
|
|
qualification_root: Path,
|
|
valid_fov_root: Path,
|
|
decoded_frames_root: Path,
|
|
timeline_path: Path,
|
|
output_root: Path,
|
|
selection: tuple[EvaluationFrameRequest, ...],
|
|
decoder_version: str,
|
|
selection_document_sha256: str,
|
|
producer_files: tuple[tuple[str, str], ...],
|
|
) -> RecordedEvaluationPack:
|
|
"""Seal exact raw/fixed-fill frames and an annotation contract for model A/B.
|
|
|
|
The pack is immutable. Human annotations are authored in a separate working
|
|
copy of ``annotation-template.json`` and later sealed as their own generation.
|
|
This keeps source imagery and the evolving review state from being conflated.
|
|
"""
|
|
|
|
job = validate_camera_compute_job(job_root)
|
|
qualification = validate_recorded_qualification_slice(
|
|
qualification_root,
|
|
job_root=job.job_root,
|
|
)
|
|
valid_fov = validate_k1_valid_fov_mask(valid_fov_root)
|
|
if not isinstance(decoder_version, str) or not 1 <= len(decoder_version) <= 256:
|
|
raise RecordedEvaluationPackError("decoder version is invalid")
|
|
if _SHA256.fullmatch(selection_document_sha256) is None:
|
|
raise RecordedEvaluationPackError("selection document SHA-256 is invalid")
|
|
producer_identity = _validate_producer_files(producer_files)
|
|
if (
|
|
valid_fov.source_id != job.source_id
|
|
or valid_fov.width != 800
|
|
or valid_fov.height != 600
|
|
):
|
|
raise RecordedEvaluationPackError("valid-FOV does not match the camera job")
|
|
selected = _validate_selection(selection, job.segment_count, qualification)
|
|
timestamps = _read_timeline(
|
|
timeline_path,
|
|
job.segment_count,
|
|
job.timeline_start_seconds,
|
|
job.timeline_end_seconds,
|
|
)
|
|
frames_root = decoded_frames_root.expanduser().resolve(strict=True)
|
|
if not frames_root.is_dir():
|
|
raise RecordedEvaluationPackError("decoded frame root is not a directory")
|
|
with Image.open(valid_fov.mask_path) as opened_mask:
|
|
mask = opened_mask.copy()
|
|
if mask.mode != "L" or mask.size != (valid_fov.width, valid_fov.height):
|
|
raise RecordedEvaluationPackError("valid-FOV mask cannot be applied to the frames")
|
|
|
|
source_by_index = {frame.frame_index: frame for frame in qualification.frames}
|
|
job_index = _read_job_index(job.job_root, job.source_id, job.codec_epoch)
|
|
decoded: list[dict[str, Any]] = []
|
|
for image_id, request in enumerate(selected, start=1):
|
|
source = source_by_index.get(request.frame_index)
|
|
row = job_index[request.frame_index]
|
|
sequence = _nonnegative_integer(row.get("sequence"), "segment sequence")
|
|
segment_sha256 = row.get("sha256")
|
|
if (
|
|
sequence != request.frame_index + 1
|
|
or not isinstance(segment_sha256, str)
|
|
or _SHA256.fullmatch(segment_sha256) is None
|
|
or (source is not None and source.segment_sha256 != segment_sha256)
|
|
):
|
|
raise RecordedEvaluationPackError("selected frame identity changed")
|
|
source_path = frames_root / f"frame-{request.frame_index:06d}.png"
|
|
_confined_regular_file(source_path, frames_root)
|
|
try:
|
|
with Image.open(source_path) as opened:
|
|
raw = opened.convert("RGB")
|
|
except OSError as exc:
|
|
raise RecordedEvaluationPackError("decoded evaluation frame is unavailable") from exc
|
|
if raw.size != (valid_fov.width, valid_fov.height):
|
|
raise RecordedEvaluationPackError("decoded evaluation frame dimensions changed")
|
|
fill = Image.composite(raw, Image.new("RGB", raw.size, (0, 0, 0)), mask)
|
|
decoded.append(
|
|
{
|
|
"image_id": image_id,
|
|
"request": request,
|
|
"sequence": sequence,
|
|
"segment_sha256": segment_sha256,
|
|
"session_seconds": timestamps[request.frame_index],
|
|
"raw": raw,
|
|
"fill": fill,
|
|
"raw_rgb_sha256": _pixel_sha256(raw),
|
|
"fill_rgb_sha256": _pixel_sha256(fill),
|
|
}
|
|
)
|
|
|
|
identity = {
|
|
"schema_version": EVALUATION_PACK_IDENTITY_SCHEMA,
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": job.session_id,
|
|
"source_id": job.source_id,
|
|
"codec_epoch": job.codec_epoch,
|
|
"qualification_generation_id": qualification.generation_id,
|
|
"qualification_policy": qualification.policy,
|
|
"valid_fov_generation_id": valid_fov.generation_id,
|
|
"calibration_sha256": valid_fov.calibration_sha256,
|
|
"calibration_slot": valid_fov.calibration_slot,
|
|
"selection_policy": SELECTION_POLICY,
|
|
"selection_document_sha256": selection_document_sha256,
|
|
"preprocessing_profile": "fixed-valid-fov-fill/v1",
|
|
"decoder_version": decoder_version,
|
|
"producer_files": producer_identity,
|
|
"resolution": [valid_fov.width, valid_fov.height],
|
|
"frames": [
|
|
{
|
|
"image_id": item["image_id"],
|
|
"frame_index": item["request"].frame_index,
|
|
"sequence": item["sequence"],
|
|
"segment_sha256": item["segment_sha256"],
|
|
"session_seconds": item["session_seconds"],
|
|
"role": item["request"].role,
|
|
"group_id": item["request"].group_id,
|
|
"raw_rgb_sha256": item["raw_rgb_sha256"],
|
|
"valid_fov_fill_rgb_sha256": item["fill_rgb_sha256"],
|
|
}
|
|
for item in decoded
|
|
],
|
|
}
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
generation_id = f"evaluation-pack-{identity_sha256}"
|
|
root = _prepare_private_directory(output_root)
|
|
final = root / generation_id
|
|
if final.exists():
|
|
existing = validate_recorded_evaluation_pack(
|
|
final,
|
|
job_root=job.job_root,
|
|
qualification_root=qualification.root,
|
|
valid_fov_root=valid_fov.root,
|
|
)
|
|
if existing.input_sha256 != job.input_sha256:
|
|
raise RecordedEvaluationPackError("evaluation generation collides with another input")
|
|
return existing
|
|
|
|
staging = root / f".{generation_id}.{secrets.token_hex(12)}.incomplete"
|
|
published = False
|
|
try:
|
|
(staging / "images" / "raw").mkdir(mode=0o700, parents=True)
|
|
(staging / "images" / "valid-fov-fill").mkdir(mode=0o700, parents=True)
|
|
artifacts: list[dict[str, Any]] = []
|
|
for item in decoded:
|
|
image_id = int(item["image_id"])
|
|
frame_index = item["request"].frame_index
|
|
name = f"image-{image_id:03d}-frame-{frame_index:06d}.png"
|
|
raw_path = staging / "images" / "raw" / name
|
|
fill_path = staging / "images" / "valid-fov-fill" / name
|
|
item["raw"].save(raw_path, format="PNG", optimize=False)
|
|
item["fill"].save(fill_path, format="PNG", optimize=False)
|
|
_fsync_file(raw_path)
|
|
_fsync_file(fill_path)
|
|
os.chmod(raw_path, 0o600)
|
|
os.chmod(fill_path, 0o600)
|
|
artifacts.extend((_artifact(raw_path, staging), _artifact(fill_path, staging)))
|
|
|
|
contract = _annotation_contract(identity_sha256)
|
|
template = _annotation_template(generation_id, identity["frames"])
|
|
contract_path = staging / "annotation-contract.json"
|
|
template_path = staging / "annotation-template.json"
|
|
write_json_atomic(contract_path, contract)
|
|
write_json_atomic(template_path, template)
|
|
os.chmod(contract_path, 0o600)
|
|
os.chmod(template_path, 0o600)
|
|
artifacts.extend((_artifact(contract_path, staging), _artifact(template_path, staging)))
|
|
manifest = {
|
|
"schema_version": EVALUATION_PACK_SCHEMA,
|
|
"generation_id": generation_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": utc_now_iso(),
|
|
"artifacts": artifacts,
|
|
"annotation_state": {
|
|
"state": "unannotated",
|
|
"mutable_inside_pack": False,
|
|
"workflow": (
|
|
"copy annotation-template.json to a review workspace; seal reviewed "
|
|
"annotations as a separate content-addressed ground-truth generation"
|
|
),
|
|
},
|
|
}
|
|
write_json_atomic(staging / "manifest.json", manifest)
|
|
os.chmod(staging / "manifest.json", 0o600)
|
|
_fsync_tree(staging)
|
|
os.replace(staging, final)
|
|
_fsync_directory(root)
|
|
published = True
|
|
finally:
|
|
if not published and staging.exists():
|
|
shutil.rmtree(staging)
|
|
|
|
return validate_recorded_evaluation_pack(
|
|
final,
|
|
job_root=job.job_root,
|
|
qualification_root=qualification.root,
|
|
valid_fov_root=valid_fov.root,
|
|
)
|
|
|
|
|
|
def validate_recorded_evaluation_pack(
|
|
pack_root: Path,
|
|
*,
|
|
job_root: Path,
|
|
qualification_root: Path,
|
|
valid_fov_root: Path,
|
|
) -> RecordedEvaluationPack:
|
|
root = pack_root.expanduser().resolve(strict=True)
|
|
if not root.is_dir() or _SAFE_GENERATION.fullmatch(root.name) is None:
|
|
raise RecordedEvaluationPackError("evaluation pack root is invalid")
|
|
manifest = _read_json_object(root / "manifest.json", root, MAX_MANIFEST_BYTES)
|
|
identity = manifest.get("identity")
|
|
identity_sha256 = manifest.get("identity_sha256")
|
|
if (
|
|
manifest.get("schema_version") != EVALUATION_PACK_SCHEMA
|
|
or manifest.get("generation_id") != root.name
|
|
or not isinstance(identity, dict)
|
|
or identity.get("schema_version") != EVALUATION_PACK_IDENTITY_SCHEMA
|
|
or not isinstance(identity_sha256, str)
|
|
or _SHA256.fullmatch(identity_sha256) is None
|
|
or root.name != f"evaluation-pack-{identity_sha256}"
|
|
or hashlib.sha256(_canonical_json(identity)).hexdigest() != identity_sha256
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation pack identity is inconsistent")
|
|
job = validate_camera_compute_job(job_root)
|
|
qualification = validate_recorded_qualification_slice(
|
|
qualification_root,
|
|
job_root=job.job_root,
|
|
)
|
|
valid_fov = validate_k1_valid_fov_mask(valid_fov_root)
|
|
if (
|
|
identity.get("job_id") != job.job_id
|
|
or identity.get("input_sha256") != job.input_sha256
|
|
or identity.get("session_id") != job.session_id
|
|
or identity.get("source_id") != job.source_id
|
|
or identity.get("codec_epoch") != job.codec_epoch
|
|
or identity.get("qualification_generation_id") != qualification.generation_id
|
|
or identity.get("qualification_policy") != qualification.policy
|
|
or identity.get("valid_fov_generation_id") != valid_fov.generation_id
|
|
or identity.get("calibration_sha256") != valid_fov.calibration_sha256
|
|
or identity.get("calibration_slot") != valid_fov.calibration_slot
|
|
or identity.get("selection_policy") != SELECTION_POLICY
|
|
or not isinstance(identity.get("selection_document_sha256"), str)
|
|
or _SHA256.fullmatch(str(identity["selection_document_sha256"])) is None
|
|
or identity.get("preprocessing_profile") != "fixed-valid-fov-fill/v1"
|
|
or not isinstance(identity.get("decoder_version"), str)
|
|
or not 1 <= len(str(identity["decoder_version"])) <= 256
|
|
or identity.get("resolution") != [valid_fov.width, valid_fov.height]
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation pack bindings changed")
|
|
producer_files = identity.get("producer_files")
|
|
if not isinstance(producer_files, list):
|
|
raise RecordedEvaluationPackError("evaluation producer identity is unavailable")
|
|
normalized_producer = _validate_producer_files(
|
|
tuple(
|
|
(str(row.get("name")), str(row.get("sha256")))
|
|
for row in producer_files
|
|
if isinstance(row, dict)
|
|
)
|
|
)
|
|
if producer_files != normalized_producer:
|
|
raise RecordedEvaluationPackError("evaluation producer identity changed")
|
|
rows = identity.get("frames")
|
|
if (
|
|
not isinstance(rows, list)
|
|
or not MIN_EVALUATION_FRAMES <= len(rows) <= MAX_EVALUATION_FRAMES
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation pack frame set is invalid")
|
|
artifact_rows = manifest.get("artifacts")
|
|
if not isinstance(artifact_rows, list) or len(artifact_rows) != len(rows) * 2 + 2:
|
|
raise RecordedEvaluationPackError("evaluation pack artifact set is invalid")
|
|
artifacts: dict[str, dict[str, Any]] = {}
|
|
for artifact in artifact_rows:
|
|
if not isinstance(artifact, dict) or not isinstance(artifact.get("path"), str):
|
|
raise RecordedEvaluationPackError("evaluation artifact descriptor is invalid")
|
|
path_text = str(artifact["path"])
|
|
if path_text in artifacts:
|
|
raise RecordedEvaluationPackError("evaluation artifact is duplicated")
|
|
path = root / path_text
|
|
metadata = _confined_regular_file(path, root)
|
|
digest = artifact.get("sha256")
|
|
if (
|
|
artifact.get("byte_length") != metadata.st_size
|
|
or not isinstance(digest, str)
|
|
or _SHA256.fullmatch(digest) is None
|
|
or _sha256_file(path) != digest
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation artifact changed")
|
|
artifacts[path_text] = artifact
|
|
contract_path = root / "annotation-contract.json"
|
|
template_path = root / "annotation-template.json"
|
|
contract = _read_json_object(contract_path, root, MAX_MANIFEST_BYTES)
|
|
template = _read_json_object(template_path, root, MAX_MANIFEST_BYTES)
|
|
if (
|
|
contract.get("schema_version") != ANNOTATION_CONTRACT_SCHEMA
|
|
or contract.get("evaluation_identity_sha256") != identity_sha256
|
|
or template.get("schema_version") != ANNOTATION_TEMPLATE_SCHEMA
|
|
or template.get("evaluation_pack_id") != root.name
|
|
or template.get("images") != rows
|
|
or "annotation-contract.json" not in artifacts
|
|
or "annotation-template.json" not in artifacts
|
|
):
|
|
raise RecordedEvaluationPackError("annotation contract changed")
|
|
|
|
qualification_indices = {frame.frame_index for frame in qualification.frames}
|
|
job_index = _read_job_index(job.job_root, job.source_id, job.codec_epoch)
|
|
with Image.open(valid_fov.mask_path) as opened_mask:
|
|
validation_mask = opened_mask.copy()
|
|
frames: list[EvaluationPackFrame] = []
|
|
previous_index = -1
|
|
temporal_groups: dict[str, list[int]] = {}
|
|
for position, row in enumerate(rows, start=1):
|
|
if not isinstance(row, dict):
|
|
raise RecordedEvaluationPackError("evaluation frame descriptor is invalid")
|
|
frame_index = row.get("frame_index")
|
|
image_id = row.get("image_id")
|
|
role = row.get("role")
|
|
group_id = row.get("group_id")
|
|
session_seconds = row.get("session_seconds")
|
|
if (
|
|
image_id != position
|
|
or not isinstance(frame_index, int)
|
|
or isinstance(frame_index, bool)
|
|
or not previous_index < frame_index < job.segment_count
|
|
or role not in {"anchor", "temporal"}
|
|
or not isinstance(group_id, str)
|
|
or _SAFE_GROUP.fullmatch(group_id) is None
|
|
or not isinstance(session_seconds, (int, float))
|
|
or isinstance(session_seconds, bool)
|
|
or not math.isfinite(float(session_seconds))
|
|
or not job.timeline_start_seconds <= float(session_seconds) <= job.timeline_end_seconds
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation frame ordering is invalid")
|
|
if role == "anchor" and frame_index not in qualification_indices:
|
|
raise RecordedEvaluationPackError("anchor escaped the qualification slice")
|
|
if role == "temporal":
|
|
temporal_groups.setdefault(group_id, []).append(frame_index)
|
|
name = f"image-{image_id:03d}-frame-{frame_index:06d}.png"
|
|
raw_relative = f"images/raw/{name}"
|
|
fill_relative = f"images/valid-fov-fill/{name}"
|
|
if raw_relative not in artifacts or fill_relative not in artifacts:
|
|
raise RecordedEvaluationPackError("evaluation image artifacts are incomplete")
|
|
raw_path = root / raw_relative
|
|
fill_path = root / fill_relative
|
|
with Image.open(raw_path) as raw_opened, Image.open(fill_path) as fill_opened:
|
|
raw = raw_opened.convert("RGB")
|
|
fill = fill_opened.convert("RGB")
|
|
expected_fill = Image.composite(
|
|
raw,
|
|
Image.new("RGB", raw.size, (0, 0, 0)),
|
|
validation_mask,
|
|
)
|
|
expected_segment_sha256 = job_index[frame_index].get("sha256")
|
|
if (
|
|
raw.size != (valid_fov.width, valid_fov.height)
|
|
or fill.size != raw.size
|
|
or _pixel_sha256(raw) != row.get("raw_rgb_sha256")
|
|
or _pixel_sha256(fill) != row.get("valid_fov_fill_rgb_sha256")
|
|
or _pixel_sha256(expected_fill) != row.get("valid_fov_fill_rgb_sha256")
|
|
or not isinstance(row.get("segment_sha256"), str)
|
|
or _SHA256.fullmatch(str(row["segment_sha256"])) is None
|
|
or row.get("segment_sha256") != expected_segment_sha256
|
|
or row.get("sequence") != frame_index + 1
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation frame pixels or identity changed")
|
|
frames.append(
|
|
EvaluationPackFrame(
|
|
image_id=image_id,
|
|
frame_index=frame_index,
|
|
sequence=frame_index + 1,
|
|
segment_sha256=str(row["segment_sha256"]),
|
|
session_seconds=float(session_seconds),
|
|
role=role,
|
|
group_id=group_id,
|
|
raw_path=raw_path,
|
|
valid_fov_fill_path=fill_path,
|
|
)
|
|
)
|
|
previous_index = frame_index
|
|
for group in temporal_groups.values():
|
|
if len(group) < 3 or any(
|
|
right != left + 1 for left, right in zip(group, group[1:], strict=False)
|
|
):
|
|
raise RecordedEvaluationPackError("temporal group is not a consecutive clip")
|
|
if len(temporal_groups) < 2:
|
|
raise RecordedEvaluationPackError("evaluation pack lacks temporal coverage")
|
|
return RecordedEvaluationPack(
|
|
generation_id=root.name,
|
|
root=root,
|
|
manifest_path=root / "manifest.json",
|
|
annotation_contract_path=contract_path,
|
|
annotation_template_path=template_path,
|
|
job_id=job.job_id,
|
|
input_sha256=job.input_sha256,
|
|
qualification_generation_id=qualification.generation_id,
|
|
valid_fov_generation_id=valid_fov.generation_id,
|
|
calibration_sha256=valid_fov.calibration_sha256,
|
|
frames=tuple(frames),
|
|
)
|
|
|
|
|
|
def _validate_selection(
|
|
selection: tuple[EvaluationFrameRequest, ...],
|
|
source_count: int,
|
|
qualification: Any,
|
|
) -> tuple[EvaluationFrameRequest, ...]:
|
|
if not MIN_EVALUATION_FRAMES <= len(selection) <= MAX_EVALUATION_FRAMES:
|
|
raise RecordedEvaluationPackError("evaluation selection size is outside bounds")
|
|
qualification_indices = {frame.frame_index for frame in qualification.frames}
|
|
previous = -1
|
|
temporal_groups: dict[str, list[int]] = {}
|
|
anchors = 0
|
|
for request in selection:
|
|
if (
|
|
not isinstance(request, EvaluationFrameRequest)
|
|
or not isinstance(request.frame_index, int)
|
|
or isinstance(request.frame_index, bool)
|
|
or not previous < request.frame_index < source_count
|
|
or request.role not in {"anchor", "temporal"}
|
|
or _SAFE_GROUP.fullmatch(request.group_id) is None
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation selection is invalid")
|
|
if request.role == "anchor":
|
|
if request.frame_index not in qualification_indices:
|
|
raise RecordedEvaluationPackError("anchor is outside the qualification slice")
|
|
anchors += 1
|
|
else:
|
|
temporal_groups.setdefault(request.group_id, []).append(request.frame_index)
|
|
previous = request.frame_index
|
|
if anchors < MIN_EVALUATION_FRAMES:
|
|
raise RecordedEvaluationPackError("evaluation selection has too few anchors")
|
|
if len(temporal_groups) < 2:
|
|
raise RecordedEvaluationPackError("evaluation selection needs at least two temporal clips")
|
|
for group in temporal_groups.values():
|
|
if len(group) < 3 or any(
|
|
right != left + 1 for left, right in zip(group, group[1:], strict=False)
|
|
):
|
|
raise RecordedEvaluationPackError("temporal clip frames must be consecutive")
|
|
return selection
|
|
|
|
|
|
def _annotation_contract(evaluation_identity_sha256: str) -> dict[str, Any]:
|
|
categories = [
|
|
(1, "person", "thing"),
|
|
(2, "bicycle", "thing"),
|
|
(3, "motorcycle", "thing"),
|
|
(4, "car", "thing"),
|
|
(5, "heavy_vehicle", "thing"),
|
|
(6, "building_structure", "stuff"),
|
|
(7, "paved_road", "stuff"),
|
|
(8, "sidewalk_curb", "stuff"),
|
|
(9, "ground_dirt", "stuff"),
|
|
(10, "grass_low_vegetation", "stuff"),
|
|
(11, "tree_woody_vegetation", "stuff"),
|
|
(12, "sky", "stuff"),
|
|
(13, "static_obstacle", "thing"),
|
|
(14, "animal", "thing"),
|
|
(15, "other_background", "stuff"),
|
|
]
|
|
return {
|
|
"schema_version": ANNOTATION_CONTRACT_SCHEMA,
|
|
"evaluation_identity_sha256": evaluation_identity_sha256,
|
|
"task": "2d-panoptic-and-instance-qualification",
|
|
"categories": [
|
|
{"id": category_id, "name": name, "kind": kind}
|
|
for category_id, name, kind in categories
|
|
],
|
|
"label_map": {
|
|
"outside_valid_fov": 0,
|
|
"ambiguous_or_unresolvable": 255,
|
|
"metric_category_ids": [category_id for category_id, _name, _kind in categories],
|
|
},
|
|
"policy": {
|
|
"coverage": "label every resolvable pixel inside the fixed valid-FOV mask",
|
|
"instances": (
|
|
"annotate every identifiable thing instance; preserve occluded and truncated "
|
|
"flags; use ambiguous=255 only when class or boundary cannot be resolved"
|
|
),
|
|
"lens_exterior": "must remain label 0 and is excluded from all accuracy metrics",
|
|
"prelabels": (
|
|
"model-assisted prelabels are permitted only as drafts; every accepted object "
|
|
"and semantic region requires human review"
|
|
),
|
|
"review": "two-pass review: annotator complete, then reviewer accepted",
|
|
},
|
|
"required_metrics": {
|
|
"semantic": ["per_class_iou", "macro_miou_present", "frequency_weighted_iou"],
|
|
"instance": ["ap_50_95", "ap50", "ap75", "ar100", "per_class_recall"],
|
|
"safety_proxies": [
|
|
"person_vehicle_miss_rate",
|
|
"false_large_instance_rate",
|
|
"valid_fov_boundary_leakage",
|
|
],
|
|
"temporal": ["class_flicker_rate", "instance_id_switches", "mask_iou_jitter"],
|
|
},
|
|
"acceptance": {
|
|
"unreviewed_frames": 0,
|
|
"pixels_outside_valid_fov_nonzero": 0,
|
|
"invalid_category_pixels": 0,
|
|
"duplicate_instance_ids_per_frame": 0,
|
|
"metric_reporting": "mean, p50, p95 and worst-frame identities where applicable",
|
|
},
|
|
}
|
|
|
|
|
|
def _validate_producer_files(files: tuple[tuple[str, str], ...]) -> list[dict[str, str]]:
|
|
if not 1 <= len(files) <= 8:
|
|
raise RecordedEvaluationPackError("evaluation producer file set is invalid")
|
|
normalized: list[dict[str, str]] = []
|
|
seen: set[str] = set()
|
|
for name, digest in files:
|
|
if (
|
|
not isinstance(name, str)
|
|
or _SAFE_GROUP.fullmatch(name) is None
|
|
or name in seen
|
|
or not isinstance(digest, str)
|
|
or _SHA256.fullmatch(digest) is None
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation producer file identity is invalid")
|
|
normalized.append({"name": name, "sha256": digest})
|
|
seen.add(name)
|
|
return normalized
|
|
|
|
|
|
def _annotation_template(generation_id: str, frames: Any) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": ANNOTATION_TEMPLATE_SCHEMA,
|
|
"evaluation_pack_id": generation_id,
|
|
"state": "unannotated",
|
|
"images": frames,
|
|
"semantic_masks": [],
|
|
"instances": [],
|
|
"reviews": [
|
|
{
|
|
"image_id": row["image_id"],
|
|
"annotation_status": "unannotated",
|
|
"review_status": "unreviewed",
|
|
"notes": "",
|
|
}
|
|
for row in frames
|
|
],
|
|
}
|
|
|
|
|
|
def _read_timeline(path: Path, count: int, start: float, end: float) -> list[float]:
|
|
root = path.expanduser().resolve(strict=True).parent
|
|
resolved = path.expanduser().resolve(strict=True)
|
|
_confined_regular_file(resolved, root)
|
|
values: list[float] = []
|
|
try:
|
|
with resolved.open("r", encoding="utf-8") as stream:
|
|
for expected, line in enumerate(stream):
|
|
if len(line.encode("utf-8")) > MAX_TIMELINE_LINE_BYTES:
|
|
raise RecordedEvaluationPackError("decoded timeline row is too large")
|
|
row = json.loads(line)
|
|
value = row.get("session_seconds") if isinstance(row, dict) else None
|
|
if row.get("frame_index") != expected or not isinstance(value, (int, float)):
|
|
raise RecordedEvaluationPackError("decoded timeline order changed")
|
|
timestamp = float(value)
|
|
if not math.isfinite(timestamp) or not start <= timestamp <= end:
|
|
raise RecordedEvaluationPackError("decoded timeline escapes the job")
|
|
if values and timestamp <= values[-1]:
|
|
raise RecordedEvaluationPackError("decoded timeline is not monotonic")
|
|
values.append(timestamp)
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError, AttributeError) as exc:
|
|
raise RecordedEvaluationPackError("decoded timeline is unavailable") from exc
|
|
if len(values) != count:
|
|
raise RecordedEvaluationPackError("decoded timeline frame count changed")
|
|
return values
|
|
|
|
|
|
def _read_job_index(job_root: Path, source_id: str, codec_epoch: int) -> list[dict[str, Any]]:
|
|
path = job_root / "input" / "camera" / source_id / f"epoch-{codec_epoch}" / "index.jsonl"
|
|
rows: list[dict[str, Any]] = []
|
|
try:
|
|
with path.open("r", encoding="utf-8") as stream:
|
|
for line in stream:
|
|
row = json.loads(line)
|
|
if not isinstance(row, dict):
|
|
raise RecordedEvaluationPackError("camera index row is invalid")
|
|
rows.append(row)
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RecordedEvaluationPackError("camera index is unavailable") from exc
|
|
if not rows:
|
|
raise RecordedEvaluationPackError("camera index is empty")
|
|
return rows
|
|
|
|
|
|
def _nonnegative_integer(value: Any, label: str) -> int:
|
|
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
|
raise RecordedEvaluationPackError(f"{label} is invalid")
|
|
return value
|
|
|
|
|
|
def _pixel_sha256(image: Image.Image) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(image.width.to_bytes(4, "big"))
|
|
digest.update(image.height.to_bytes(4, "big"))
|
|
digest.update(image.mode.encode("ascii"))
|
|
digest.update(b"\x00")
|
|
digest.update(image.tobytes())
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _artifact(path: Path, root: Path) -> dict[str, Any]:
|
|
return {
|
|
"path": path.relative_to(root).as_posix(),
|
|
"byte_length": path.stat().st_size,
|
|
"sha256": _sha256_file(path),
|
|
}
|
|
|
|
|
|
def _prepare_private_directory(path: Path) -> Path:
|
|
candidate = path.expanduser()
|
|
candidate.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
metadata = candidate.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
|
|
raise RecordedEvaluationPackError("evaluation output root must be a real directory")
|
|
root = candidate.resolve(strict=True)
|
|
os.chmod(root, 0o700)
|
|
return root
|
|
|
|
|
|
def _read_json_object(path: Path, root: Path, maximum_bytes: int) -> dict[str, Any]:
|
|
metadata = _confined_regular_file(path, root)
|
|
if not 0 < metadata.st_size <= maximum_bytes:
|
|
raise RecordedEvaluationPackError("evaluation JSON is outside bounds")
|
|
try:
|
|
value = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
raise RecordedEvaluationPackError("evaluation JSON is unavailable") from exc
|
|
if not isinstance(value, dict):
|
|
raise RecordedEvaluationPackError("evaluation JSON is not an object")
|
|
return value
|
|
|
|
|
|
def _confined_regular_file(path: Path, root: Path) -> os.stat_result:
|
|
try:
|
|
resolved_root = root.resolve(strict=True)
|
|
resolved = path.resolve(strict=True)
|
|
metadata = path.lstat()
|
|
except OSError as exc:
|
|
raise RecordedEvaluationPackError("evaluation artifact is unavailable") from exc
|
|
if (
|
|
stat.S_ISLNK(metadata.st_mode)
|
|
or not stat.S_ISREG(metadata.st_mode)
|
|
or not resolved.is_relative_to(resolved_root)
|
|
):
|
|
raise RecordedEvaluationPackError("evaluation artifact is not confined")
|
|
return metadata
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _canonical_json(value: object) -> bytes:
|
|
try:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
except (TypeError, ValueError) as exc:
|
|
raise SessionIntegrityError("evaluation identity is not canonical JSON") from exc
|
|
|
|
|
|
def _fsync_file(path: Path) -> None:
|
|
with path.open("rb") as stream:
|
|
os.fsync(stream.fileno())
|
|
|
|
|
|
def _fsync_tree(root: Path) -> None:
|
|
for path in sorted(root.rglob("*"), reverse=True):
|
|
if path.is_file():
|
|
_fsync_file(path)
|
|
elif path.is_dir():
|
|
_fsync_directory(path)
|
|
_fsync_directory(root)
|
|
|
|
|
|
def _fsync_directory(path: Path) -> None:
|
|
flags = os.O_RDONLY
|
|
if hasattr(os, "O_DIRECTORY"):
|
|
flags |= os.O_DIRECTORY
|
|
descriptor = os.open(path, flags)
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|