941 lines
36 KiB
Python
941 lines
36 KiB
Python
"""Immutable LAB catalog projections for accepted integrated perception runs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from k1link.artifacts import write_json_atomic
|
|
from k1link.sessions import (
|
|
LabSessionBinding,
|
|
SessionIntegrityError,
|
|
SessionStore,
|
|
publish_lab_replay_cache,
|
|
)
|
|
|
|
from .integrated_perception import (
|
|
IntegratedPerceptionResult,
|
|
validate_integrated_perception_result,
|
|
)
|
|
from .jobs import CameraComputeJob, validate_camera_compute_job
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedIntegratedLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
|
|
|
|
def publish_integrated_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
source_result_root: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
provenance: dict[str, Any] | None = None,
|
|
) -> PublishedIntegratedLabInstance:
|
|
"""Snapshot one accepted visual result as a separately replayable LAB run.
|
|
|
|
Large immutable camera, LiDAR, array and RRD payloads are hard-linked.
|
|
Content-addressed manifests are rebound to the LAB session id, so the
|
|
existing viewer resolves one exact result instead of selecting whichever
|
|
source-session result happens to be newest.
|
|
"""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
source_result_path = source_result_root.expanduser().resolve(strict=True)
|
|
source_document = _read_object(source_result_path / "result.json", source_result_path)
|
|
identity = source_document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError("integrated LAB source has no job identity")
|
|
source_job_root = jobs_root / identity["job_id"]
|
|
source = validate_integrated_perception_result(
|
|
source_job_root,
|
|
source_result_path,
|
|
packs_root,
|
|
)
|
|
if not source.accepted:
|
|
raise SessionIntegrityError("only accepted integrated results can become LAB instances")
|
|
|
|
lab_job = _publish_lab_job(source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(source, lab_job, packs_root, lab_session_id)
|
|
lab_result = _publish_lab_result(
|
|
source,
|
|
lab_job,
|
|
lab_pack,
|
|
results_root,
|
|
lab_session_id,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
lab_result,
|
|
packs_root,
|
|
)
|
|
store = SessionStore(root)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source.job.session_id,
|
|
lab_session_id=lab_session_id,
|
|
)
|
|
configuration = source_document["identity"].get("configuration")
|
|
profile_sha256 = (
|
|
configuration.get("profile_sha256") if isinstance(configuration, dict) else None
|
|
)
|
|
if not isinstance(profile_sha256, str) or len(profile_sha256) != 64:
|
|
profile_sha256 = None
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source.job.session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e10-integrated-perception",
|
|
result_id=validated.result_id,
|
|
source_result_id=source.result_id,
|
|
config_sha256=profile_sha256,
|
|
run_created_at_utc=source.created_at_utc,
|
|
provenance={
|
|
"schema_version": "missioncore.integrated-lab-publication/v1",
|
|
"storage_mode": "hard-linked-immutable-payloads",
|
|
"source_job_id": source.job.job_id,
|
|
"projected_job_id": lab_job.job_id,
|
|
"source_lidar_pack_id": source.pack_root.name,
|
|
"projected_lidar_pack_id": lab_pack.name,
|
|
**(provenance or {}),
|
|
},
|
|
)
|
|
return PublishedIntegratedLabInstance(binding=binding, job=lab_job, result=validated)
|
|
|
|
|
|
def publish_e21_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
e21_result_root: Path,
|
|
worker_result_root: Path,
|
|
semantic_reference_result_root: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedIntegratedLabInstance:
|
|
"""Publish the accepted E21 envelope as an exact 60-second visual LAB run.
|
|
|
|
E21 persisted semantic mask identities but intentionally did not duplicate
|
|
mask pixels. The immutable full-session reference contains those exact
|
|
pixels. Publication therefore materializes a bounded viewer projection only
|
|
after every E21 mask SHA-256 matches the corresponding reference mask.
|
|
Detector latest-wins replacements remain explicit empty frames.
|
|
"""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
reference_path = semantic_reference_result_root.expanduser().resolve(strict=True)
|
|
reference_document = _read_object(reference_path / "result.json", reference_path)
|
|
reference_identity = reference_document.get("identity")
|
|
if not isinstance(reference_identity, dict) or not isinstance(
|
|
reference_identity.get("job_id"), str
|
|
):
|
|
raise SessionIntegrityError("E21 semantic reference has no job identity")
|
|
reference = validate_integrated_perception_result(
|
|
jobs_root / reference_identity["job_id"],
|
|
reference_path,
|
|
packs_root,
|
|
)
|
|
if not reference.accepted or reference.source_start_frame_index != 0:
|
|
raise SessionIntegrityError("E21 semantic reference is not an accepted zero-based run")
|
|
|
|
e21_root = e21_result_root.expanduser().resolve(strict=True)
|
|
worker_root = worker_result_root.expanduser().resolve(strict=True)
|
|
e21_document, e21_report, worker_document = _validate_e21_inputs(
|
|
e21_root,
|
|
worker_root,
|
|
)
|
|
frame_count = int(e21_report["metrics"]["source"]["events_admitted"]["camera-frame"])
|
|
if frame_count < 2 or frame_count > reference.frame_count:
|
|
raise SessionIntegrityError("E21 frame count is outside the semantic reference")
|
|
|
|
lab_job = _publish_lab_job(reference.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_e21_pack(
|
|
reference,
|
|
lab_job,
|
|
packs_root,
|
|
lab_session_id,
|
|
frame_count,
|
|
)
|
|
lab_result = _publish_e21_visual_result(
|
|
reference=reference,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
lab_session_id=lab_session_id,
|
|
frame_count=frame_count,
|
|
e21_root=e21_root,
|
|
e21_document=e21_document,
|
|
e21_report=e21_report,
|
|
worker_root=worker_root,
|
|
worker_document=worker_document,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
lab_result,
|
|
packs_root,
|
|
)
|
|
store = SessionStore(root)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=reference.job.session_id,
|
|
lab_session_id=lab_session_id,
|
|
)
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=reference.job.session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e21-realtime-envelope",
|
|
result_id=validated.result_id,
|
|
source_result_id=str(e21_document["result_id"]),
|
|
config_sha256=str(e21_report["identity"]["profile_sha256"]),
|
|
run_created_at_utc=str(e21_report["created_at_utc"]),
|
|
provenance={
|
|
"schema_version": "missioncore.e21-lab-publication/v1",
|
|
"storage_mode": "hard-linked-source-and-bounded-derived-projection",
|
|
"e21_result_id": e21_document["result_id"],
|
|
"worker_result_id": worker_document["result_id"],
|
|
"semantic_reference_result_id": reference.result_id,
|
|
"semantic_mask_reconstruction": "exact-sha256-match",
|
|
"detector_replacement_frames": [
|
|
12,
|
|
239,
|
|
240,
|
|
241,
|
|
242,
|
|
433,
|
|
498,
|
|
],
|
|
"visual_frame_count": frame_count,
|
|
"source_payloads_mutated": False,
|
|
},
|
|
)
|
|
return PublishedIntegratedLabInstance(binding=binding, job=lab_job, result=validated)
|
|
|
|
|
|
def _validate_e21_inputs(
|
|
e21_root: Path,
|
|
worker_root: Path,
|
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
e21_document = _read_object(e21_root / "result.json", e21_root)
|
|
e21_report = _read_object(e21_root / "report.json", e21_root)
|
|
report_descriptor = e21_document.get("report")
|
|
if (
|
|
e21_document.get("schema_version") != "missioncore.e21-realtime-envelope-result/v1"
|
|
or e21_document.get("result_id") != e21_root.name
|
|
or e21_document.get("state") != "accepted"
|
|
or not isinstance(report_descriptor, dict)
|
|
or report_descriptor.get("path") != "report.json"
|
|
or report_descriptor.get("byte_length") != (e21_root / "report.json").stat().st_size
|
|
or report_descriptor.get("sha256") != _sha256(e21_root / "report.json")
|
|
or e21_report.get("schema_version") != "missioncore.e21-realtime-envelope-report/v1"
|
|
or e21_report.get("result_id") != e21_root.name
|
|
or e21_report.get("state") != "accepted"
|
|
or not isinstance(e21_report.get("identity"), dict)
|
|
or not isinstance(e21_report.get("metrics"), dict)
|
|
):
|
|
raise SessionIntegrityError("E21 accepted result identity is inconsistent")
|
|
|
|
worker_document = _read_object(worker_root / "result.json", worker_root)
|
|
e21_identity = e21_report["identity"]
|
|
if (
|
|
worker_document.get("schema_version") != "missioncore.e15-shadow-inference-result/v1"
|
|
or worker_document.get("result_id") != worker_root.name
|
|
or worker_document.get("result_id") != e21_identity.get("worker_result_id")
|
|
or _sha256(worker_root / "result.json") != e21_identity.get("worker_result_sha256")
|
|
or not isinstance(worker_document.get("identity"), dict)
|
|
or not isinstance(worker_document.get("artifacts"), list)
|
|
):
|
|
raise SessionIntegrityError("E21 worker result identity is inconsistent")
|
|
required = {
|
|
"e15-semantic-frames": "semantic-frames.jsonl",
|
|
"e15-fusion-frames": "fusion-frames.jsonl",
|
|
"e15-world-state": "world-state.jsonl",
|
|
"worker-gpu-telemetry": "gpu-telemetry.jsonl",
|
|
}
|
|
descriptors = {
|
|
value.get("kind"): value
|
|
for value in worker_document["artifacts"]
|
|
if isinstance(value, dict) and value.get("kind") in required
|
|
}
|
|
if set(descriptors) != set(required):
|
|
raise SessionIntegrityError("E21 worker artifacts are incomplete")
|
|
for kind, name in required.items():
|
|
descriptor = descriptors[kind]
|
|
path = worker_root / name
|
|
if (
|
|
descriptor.get("path") != name
|
|
or descriptor.get("byte_length") != path.stat().st_size
|
|
or descriptor.get("sha256") != _sha256(path)
|
|
):
|
|
raise SessionIntegrityError("E21 worker artifact identity changed")
|
|
return e21_document, e21_report, worker_document
|
|
|
|
|
|
def _publish_e21_pack(
|
|
reference: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
packs_root: Path,
|
|
lab_session_id: str,
|
|
frame_count: int,
|
|
) -> Path:
|
|
source_manifest = _read_object(reference.pack_root / "manifest.json", reference.pack_root)
|
|
with np.load(reference.pack_root / "lidar-pack.npz", allow_pickle=False) as arrays:
|
|
cloud_end = int(arrays["cloud_offsets"][frame_count])
|
|
payload = {
|
|
"frame_indices": arrays["frame_indices"][:frame_count].copy(),
|
|
"source_frame_indices": arrays["source_frame_indices"][:frame_count].copy(),
|
|
"session_seconds": arrays["session_seconds"][:frame_count].copy(),
|
|
"sample_available": arrays["sample_available"][:frame_count].copy(),
|
|
"cloud_offsets": arrays["cloud_offsets"][: frame_count + 1].copy(),
|
|
"cloud_points_map": arrays["cloud_points_map"][:cloud_end].copy(),
|
|
"pose_positions_map": arrays["pose_positions_map"][:frame_count].copy(),
|
|
"pose_quaternions_map_from_lidar": arrays[
|
|
"pose_quaternions_map_from_lidar"
|
|
][:frame_count].copy(),
|
|
"lidar_camera_delta_ms": arrays["lidar_camera_delta_ms"][:frame_count].copy(),
|
|
"pose_point_delta_ms": arrays["pose_point_delta_ms"][:frame_count].copy(),
|
|
"intrinsic_fx_fy_cx_cy": arrays["intrinsic_fx_fy_cx_cy"].copy(),
|
|
"distortion_kb4": arrays["distortion_kb4"].copy(),
|
|
"t_camera_from_lidar": arrays["t_camera_from_lidar"].copy(),
|
|
}
|
|
identity = json.loads(json.dumps(source_manifest["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"frame_count": frame_count,
|
|
"source_start_frame_index": 0,
|
|
"source_end_frame_index": frame_count - 1,
|
|
"available_lidar_frames": int(np.count_nonzero(payload["sample_available"])),
|
|
"point_count": int(payload["cloud_points_map"].shape[0]),
|
|
"timeline_start_seconds": float(payload["session_seconds"][0]),
|
|
"timeline_end_seconds": float(payload["session_seconds"][-1]),
|
|
"visual_projection": "accepted-e21-envelope/v1",
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
pack_id = f"e10-lidar-pack-{identity_sha256}"
|
|
destination = packs_root / pack_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "manifest.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("E21 LAB LiDAR pack id collides")
|
|
return destination
|
|
staging = _staging_directory(packs_root, pack_id)
|
|
try:
|
|
arrays_path = staging / "lidar-pack.npz"
|
|
np.savez_compressed(arrays_path, **payload)
|
|
os.chmod(arrays_path, 0o600)
|
|
write_json_atomic(
|
|
staging / "manifest.json",
|
|
{
|
|
**source_manifest,
|
|
"pack_id": pack_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": str(source_manifest["created_at_utc"]),
|
|
"artifact": {
|
|
"path": arrays_path.name,
|
|
"media_type": "application/x-npz",
|
|
"byte_length": arrays_path.stat().st_size,
|
|
"sha256": _sha256(arrays_path),
|
|
},
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _publish_e21_visual_result(
|
|
*,
|
|
reference: IntegratedPerceptionResult,
|
|
lab_job: CameraComputeJob,
|
|
lab_pack: Path,
|
|
results_root: Path,
|
|
lab_session_id: str,
|
|
frame_count: int,
|
|
e21_root: Path,
|
|
e21_document: dict[str, Any],
|
|
e21_report: dict[str, Any],
|
|
worker_root: Path,
|
|
worker_document: dict[str, Any],
|
|
) -> Path:
|
|
with np.load(reference.arrays_path, allow_pickle=False) as arrays:
|
|
frame_times = arrays["frame_times_ns"][:frame_count].copy()
|
|
reference_semantic_indices = arrays["semantic_frame_indices"]
|
|
selected = reference_semantic_indices < frame_count
|
|
reference_indices = reference_semantic_indices[selected].copy()
|
|
reference_masks = arrays["semantic_masks"][selected].copy()
|
|
|
|
semantic_rows = _read_jsonl(worker_root / "semantic-frames.jsonl")
|
|
if [row.get("frame_index") for row in semantic_rows] != reference_indices.tolist():
|
|
raise SessionIntegrityError("E21 semantic frame schedule changed")
|
|
for row, mask in zip(semantic_rows, reference_masks, strict=True):
|
|
if row.get("mask_sha256") != hashlib.sha256(mask.tobytes()).hexdigest():
|
|
raise SessionIntegrityError("E21 semantic mask does not match immutable reference")
|
|
row["schema_version"] = "missioncore.e10-semantic-frame/v1"
|
|
row["session_seconds"] = float(frame_times[int(row["frame_index"])]) / 1_000_000_000
|
|
|
|
fusion_source = {
|
|
int(row["source_frame_index"]): row
|
|
for row in _read_jsonl(worker_root / "fusion-frames.jsonl")
|
|
}
|
|
world_source = {
|
|
int(row["source_frame_index"]): row
|
|
for row in _read_jsonl(worker_root / "world-state.jsonl")
|
|
}
|
|
if set(fusion_source) != set(world_source):
|
|
raise SessionIntegrityError("E21 fusion and world timelines differ")
|
|
fusion_rows: list[dict[str, Any]] = []
|
|
world_rows: list[dict[str, Any]] = []
|
|
dropped_indices: list[int] = []
|
|
for index in range(frame_count):
|
|
session_seconds = float(frame_times[index]) / 1_000_000_000
|
|
fusion = fusion_source.get(index)
|
|
world = world_source.get(index)
|
|
if fusion is None or world is None:
|
|
dropped_indices.append(index)
|
|
fusion_rows.append(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
"fusion_state": "detector-dropped-latest-wins",
|
|
"semantic_source_frame_index": None,
|
|
"semantic_status": "unavailable",
|
|
"objects": [],
|
|
}
|
|
)
|
|
world_rows.append(_dropped_world_row(index, session_seconds))
|
|
continue
|
|
normalized_fusion = json.loads(json.dumps(fusion))
|
|
normalized_fusion.update(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
normalized_world = json.loads(json.dumps(world))
|
|
normalized_world.update(
|
|
{
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
fusion_rows.append(normalized_fusion)
|
|
world_rows.append(normalized_world)
|
|
expected_drops = int(
|
|
e21_report["metrics"]["worker"]["detector"]["queue"]["dropped_overflow"]
|
|
)
|
|
if len(dropped_indices) != expected_drops:
|
|
raise SessionIntegrityError("E21 detector replacement accounting changed")
|
|
|
|
box_offsets = [0]
|
|
centers: list[list[float]] = []
|
|
half_sizes: list[list[float]] = []
|
|
quaternions: list[list[float]] = []
|
|
colors: list[list[int]] = []
|
|
for row in fusion_rows:
|
|
for item in row["objects"]:
|
|
if not str(item.get("cuboid_status", "")).startswith("accepted-"):
|
|
continue
|
|
centers.append(item["cuboid_center_map"])
|
|
half_sizes.append(item["cuboid_half_size"])
|
|
quaternions.append(item["cuboid_quaternion_xyzw"])
|
|
colors.append(_cuboid_color(item))
|
|
box_offsets.append(len(centers))
|
|
|
|
reference_identity = _read_object(
|
|
reference.result_root / "result.json",
|
|
reference.result_root,
|
|
)["identity"]
|
|
worker_identity = worker_document["identity"]
|
|
configuration = {
|
|
"pipeline": "e21-shadow-realtime-envelope-visual-projection/v1",
|
|
"profile_sha256": e21_report["identity"]["profile_sha256"],
|
|
"profile": {
|
|
"source": reference_identity["configuration"]["profile"]["source"],
|
|
"replay": {
|
|
"speed": 1.0,
|
|
"bounded_latest_wins": True,
|
|
"detector_replacement_frames": dropped_indices,
|
|
},
|
|
},
|
|
"e21_result_id": e21_document["result_id"],
|
|
"worker_result_id": worker_document["result_id"],
|
|
"semantic_mask_materialization": {
|
|
"mode": "immutable-reference-exact-sha256",
|
|
"reference_result_id": reference.result_id,
|
|
"matched_masks": len(semantic_rows),
|
|
},
|
|
}
|
|
selection = {
|
|
"frame_count": frame_count,
|
|
"source_start_frame_index": 0,
|
|
"source_end_frame_index": frame_count - 1,
|
|
"timeline_start_seconds": float(frame_times[0]) / 1_000_000_000,
|
|
"timeline_end_seconds": float(frame_times[-1]) / 1_000_000_000,
|
|
"timeline_sha256": hashlib.sha256(frame_times.tobytes()).hexdigest(),
|
|
}
|
|
identity = {
|
|
"schema_version": "missioncore.e10-integrated-perception-identity/v1",
|
|
"job_id": lab_job.job_id,
|
|
"input_sha256": lab_job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"source_id": lab_job.source_id,
|
|
"lidar_pack_id": lab_pack.name,
|
|
"selection": selection,
|
|
"configuration": configuration,
|
|
"models": worker_identity["models"],
|
|
}
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
result_id = f"e10-integrated-perception-{identity_sha256}"
|
|
destination = results_root / result_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "result.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("E21 LAB visual result id collides")
|
|
return destination
|
|
|
|
staging = _staging_directory(results_root, result_id)
|
|
try:
|
|
semantic_path = staging / "semantic-frames.jsonl"
|
|
fusion_path = staging / "fusion-frames.jsonl"
|
|
world_path = staging / "world-state.jsonl"
|
|
arrays_path = staging / "transient-perception.npz"
|
|
gpu_path = staging / "gpu-telemetry.jsonl"
|
|
report_path = staging / "run-report.json"
|
|
_write_jsonl(semantic_path, semantic_rows)
|
|
_write_jsonl(fusion_path, fusion_rows)
|
|
_write_jsonl(world_path, world_rows)
|
|
np.savez_compressed(
|
|
arrays_path,
|
|
frame_times_ns=frame_times.astype(np.int64, copy=False),
|
|
semantic_frame_indices=reference_indices.astype(np.int64, copy=False),
|
|
semantic_masks=reference_masks.astype(np.uint8, copy=False),
|
|
support_offsets=np.zeros(frame_count + 1, dtype=np.int64),
|
|
support_points=np.empty((0, 3), dtype=np.float32),
|
|
support_colors=np.empty((0, 3), dtype=np.uint8),
|
|
box_offsets=np.asarray(box_offsets, dtype=np.int64),
|
|
box_centers=np.asarray(centers, dtype=np.float32).reshape((-1, 3)),
|
|
box_half_sizes=np.asarray(half_sizes, dtype=np.float32).reshape((-1, 3)),
|
|
box_quaternions=np.asarray(quaternions, dtype=np.float32).reshape((-1, 4)),
|
|
box_colors=np.asarray(colors, dtype=np.uint8).reshape((-1, 4)),
|
|
)
|
|
shutil.copyfile(worker_root / "gpu-telemetry.jsonl", gpu_path)
|
|
report = {
|
|
"schema_version": "missioncore.e10-integrated-perception-report/v1",
|
|
"result_id": result_id,
|
|
"created_at_utc": e21_report["created_at_utc"],
|
|
"state": "accepted",
|
|
"ground_truth": False,
|
|
"identity": identity,
|
|
"acceptance": {
|
|
"accepted": True,
|
|
"navigation_or_safety_accepted": False,
|
|
"checks": {
|
|
"e21_envelope_accepted": True,
|
|
"semantic_hashes_exact": True,
|
|
"detector_replacements_explicit": True,
|
|
"source_payloads_immutable": True,
|
|
},
|
|
},
|
|
"metrics": {
|
|
**e21_report["metrics"],
|
|
"visual_projection": {
|
|
"frames": frame_count,
|
|
"semantic_masks": len(semantic_rows),
|
|
"detector_replacement_frames": dropped_indices,
|
|
"accepted_cuboids": len(centers),
|
|
},
|
|
},
|
|
"runtime": _read_object(worker_root / "run-report.json", worker_root).get(
|
|
"runtime", {}
|
|
),
|
|
"limitations": [
|
|
"This is the exact accepted recorded E21 envelope, not a physical live K1 gate.",
|
|
"Seven latest-wins detector replacements are explicit empty visual frames.",
|
|
"Semantic pixels are materialized only after exact SHA-256 reference matches.",
|
|
"LiDAR support points remain in the immutable source scene and are not duplicated.",
|
|
"Navigation and safety authority remain disabled.",
|
|
],
|
|
}
|
|
write_json_atomic(report_path, report)
|
|
artifacts = [
|
|
_artifact_descriptor(
|
|
"e10-semantic-frames",
|
|
semantic_path,
|
|
"missioncore.e10-semantic-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-fusion-frames",
|
|
fusion_path,
|
|
"missioncore.e10-fusion-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-world-state",
|
|
world_path,
|
|
"missioncore.live-perception-world-state/v1",
|
|
),
|
|
_artifact_descriptor("e10-transient-perception", arrays_path, None),
|
|
_artifact_descriptor("worker-gpu-telemetry", gpu_path, None),
|
|
_artifact_descriptor(
|
|
"e10-run-report",
|
|
report_path,
|
|
"missioncore.e10-integrated-perception-report/v1",
|
|
),
|
|
]
|
|
write_json_atomic(
|
|
staging / "result.json",
|
|
{
|
|
"schema_version": "missioncore.e10-integrated-perception-result/v1",
|
|
"result_id": result_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": e21_report["created_at_utc"],
|
|
"ground_truth": False,
|
|
"publication_scope": "recorded-integrated-realtime-qualification-only",
|
|
"acceptance_state": "accepted",
|
|
"frames_processed": frame_count,
|
|
"artifacts": artifacts,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _dropped_world_row(frame_index: int, session_seconds: float) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "missioncore.live-perception-world-state/v1",
|
|
"frame_index": frame_index,
|
|
"source_frame_index": frame_index,
|
|
"session_seconds": session_seconds,
|
|
"fusion_state": "detector-dropped-latest-wins",
|
|
"object_count": 0,
|
|
"objects": [],
|
|
"delivery": {
|
|
"health": "degraded",
|
|
"semantic_status": "unavailable",
|
|
"result_age_ms": None,
|
|
"projection_status": "explicit-detector-replacement",
|
|
},
|
|
"coordinate_frames": {
|
|
"sensor_relative": "k1-lidar",
|
|
"vehicle_body": "unavailable-no-rig-to-vehicle-transform",
|
|
"world": "k1-map",
|
|
},
|
|
"clearance": {
|
|
"schema": "diagnostic-polar-obstacle-clearance/v1",
|
|
"state": "unavailable",
|
|
"frame": "k1-lidar",
|
|
"front_m": None,
|
|
"observed_sector_fraction": 0.0,
|
|
"sector_ranges_m": [None] * 72,
|
|
},
|
|
}
|
|
|
|
|
|
def _cuboid_color(item: dict[str, Any]) -> list[int]:
|
|
key = f"e10:{item.get('association_group', 'object')}:{item.get('track_id', 0)}"
|
|
digest = hashlib.sha256(key.encode()).digest()
|
|
return [64 + digest[0] % 176, 64 + digest[1] % 176, 64 + digest[2] % 176, 88]
|
|
|
|
|
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
with path.open(encoding="utf-8") as stream:
|
|
for line in stream:
|
|
value = json.loads(line)
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("LAB JSONL row is not an object")
|
|
rows.append(value)
|
|
return rows
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
|
with path.open("x", encoding="utf-8") as stream:
|
|
for row in rows:
|
|
stream.write(
|
|
json.dumps(
|
|
row,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
)
|
|
)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.chmod(path, 0o600)
|
|
|
|
|
|
def _artifact_descriptor(
|
|
kind: str,
|
|
path: Path,
|
|
schema_version: str | None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"kind": kind,
|
|
"path": path.name,
|
|
"schema_version": schema_version,
|
|
"byte_length": path.stat().st_size,
|
|
"sha256": _sha256(path),
|
|
}
|
|
|
|
|
|
def _publish_lab_job(
|
|
source: CameraComputeJob,
|
|
jobs_root: Path,
|
|
lab_session_id: str,
|
|
) -> CameraComputeJob:
|
|
manifest = _read_object(source.manifest_path, source.job_root)
|
|
input_document = json.loads(json.dumps(manifest["input"]))
|
|
input_document["session_id"] = lab_session_id
|
|
input_sha256 = hashlib.sha256(_canonical_json(input_document)).hexdigest()
|
|
job_id = f"recorded-camera-{input_sha256[:24]}"
|
|
destination = jobs_root / job_id
|
|
if destination.exists():
|
|
existing = validate_camera_compute_job(destination)
|
|
if existing.session_id != lab_session_id:
|
|
raise SessionIntegrityError("LAB compute job id collides")
|
|
return existing
|
|
staging = _staging_directory(jobs_root, job_id)
|
|
try:
|
|
_hardlink_tree(source.job_root / "input", staging / "input")
|
|
write_json_atomic(
|
|
staging / "job.json",
|
|
{
|
|
**manifest,
|
|
"job_id": job_id,
|
|
"input_sha256": input_sha256,
|
|
"input": input_document,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return validate_camera_compute_job(destination)
|
|
|
|
|
|
def _publish_lab_pack(
|
|
source: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
packs_root: Path,
|
|
lab_session_id: str,
|
|
) -> Path:
|
|
source_manifest = _read_object(source.pack_root / "manifest.json", source.pack_root)
|
|
identity = json.loads(json.dumps(source_manifest["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
pack_id = f"e10-lidar-pack-{identity_sha256}"
|
|
destination = packs_root / pack_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "manifest.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("LAB LiDAR pack id collides")
|
|
return destination
|
|
staging = _staging_directory(packs_root, pack_id)
|
|
try:
|
|
os.link(
|
|
source.pack_root / "lidar-pack.npz",
|
|
staging / "lidar-pack.npz",
|
|
follow_symlinks=False,
|
|
)
|
|
write_json_atomic(
|
|
staging / "manifest.json",
|
|
{
|
|
**source_manifest,
|
|
"pack_id": pack_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _publish_lab_result(
|
|
source: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
pack_root: Path,
|
|
results_root: Path,
|
|
lab_session_id: str,
|
|
) -> Path:
|
|
source_document = _read_object(source.result_root / "result.json", source.result_root)
|
|
identity = json.loads(json.dumps(source_document["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"lidar_pack_id": pack_root.name,
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
result_id = f"e10-integrated-perception-{identity_sha256}"
|
|
destination = results_root / result_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "result.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("LAB integrated result id collides")
|
|
return destination
|
|
staging = _staging_directory(results_root, result_id)
|
|
try:
|
|
descriptors: list[dict[str, Any]] = []
|
|
for descriptor in source_document["artifacts"]:
|
|
if descriptor["kind"] == "e10-run-report":
|
|
continue
|
|
source_path = source.result_root / descriptor["path"]
|
|
target = staging / descriptor["path"]
|
|
os.link(source_path, target, follow_symlinks=False)
|
|
descriptors.append(dict(descriptor))
|
|
|
|
report = _read_object(source.report_path, source.result_root)
|
|
report.update(
|
|
{
|
|
"result_id": result_id,
|
|
"identity": identity,
|
|
"lab_projection": {
|
|
"schema_version": "missioncore.integrated-lab-projection/v1",
|
|
"source_session_id": source.job.session_id,
|
|
"source_result_id": source.result_id,
|
|
"lab_session_id": lab_session_id,
|
|
"payloads_recomputed": False,
|
|
},
|
|
}
|
|
)
|
|
report_path = staging / "run-report.json"
|
|
write_json_atomic(report_path, report)
|
|
descriptors.append(
|
|
{
|
|
"kind": "e10-run-report",
|
|
"path": report_path.name,
|
|
"schema_version": "missioncore.e10-integrated-perception-report/v1",
|
|
"byte_length": report_path.stat().st_size,
|
|
"sha256": _sha256(report_path),
|
|
}
|
|
)
|
|
descriptor_order = {
|
|
value["kind"]: index for index, value in enumerate(source_document["artifacts"])
|
|
}
|
|
descriptors.sort(key=lambda value: descriptor_order[value["kind"]])
|
|
write_json_atomic(
|
|
staging / "result.json",
|
|
{
|
|
**source_document,
|
|
"result_id": result_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"artifacts": descriptors,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _hardlink_tree(source: Path, destination: Path) -> None:
|
|
source_root = source.resolve(strict=True)
|
|
destination.mkdir(mode=0o700, parents=True)
|
|
for current, directories, files in os.walk(source_root):
|
|
current_path = Path(current)
|
|
relative = current_path.relative_to(source_root)
|
|
target_root = destination / relative
|
|
target_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
directories.sort()
|
|
files.sort()
|
|
for filename in files:
|
|
path = current_path / filename
|
|
metadata = path.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
|
raise SessionIntegrityError("LAB source tree contains a non-regular file")
|
|
os.link(path, target_root / filename, follow_symlinks=False)
|
|
|
|
|
|
def _staging_directory(root: Path, identity: str) -> Path:
|
|
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
staging = root / f".{identity}.{secrets.token_hex(12)}.tmp"
|
|
staging.mkdir(mode=0o700)
|
|
return staging
|
|
|
|
|
|
def _publish_directory(staging: Path, destination: Path) -> None:
|
|
try:
|
|
os.replace(staging, destination)
|
|
except OSError:
|
|
if not destination.is_dir():
|
|
raise
|
|
|
|
|
|
def _remove_staging(path: Path) -> None:
|
|
if path.exists():
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def _read_object(path: Path, root: Path) -> dict[str, Any]:
|
|
resolved = path.resolve(strict=True)
|
|
if resolved.parent != root.resolve(strict=True) or resolved.is_symlink():
|
|
raise SessionIntegrityError("LAB manifest escapes its immutable root")
|
|
value = json.loads(resolved.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("LAB manifest is not a JSON object")
|
|
return value
|
|
|
|
|
|
def _canonical_json(value: object) -> bytes:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|