1975 lines
79 KiB
Python
1975 lines
79 KiB
Python
"""Immutable LAB catalog projections for accepted integrated perception runs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from k1link.artifacts import write_json_atomic
|
|
from k1link.sessions import (
|
|
LabSessionBinding,
|
|
SessionIntegrityError,
|
|
SessionStore,
|
|
publish_lab_replay_cache,
|
|
)
|
|
|
|
from .camera_ego_motion import (
|
|
CameraEgoMotionBuild,
|
|
build_camera_ego_motion_result,
|
|
)
|
|
from .inline_temporal import StreamingSemanticStabilizer, read_inline_profile
|
|
from .integrated_perception import (
|
|
IntegratedPerceptionResult,
|
|
validate_integrated_perception_result,
|
|
)
|
|
from .jobs import CameraComputeJob, validate_camera_compute_job
|
|
from .occupancy_motion import (
|
|
PersistentSupportBuild,
|
|
build_persistent_support_result,
|
|
)
|
|
from .temporal_stability import (
|
|
TemporalStabilityBuild,
|
|
_quality_metrics,
|
|
build_temporal_stability_result,
|
|
)
|
|
from .world_motion import WorldMotionBuild, build_world_motion_result
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedIntegratedLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedTemporalLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
build: TemporalStabilityBuild
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedWorldMotionLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
build: WorldMotionBuild
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedPersistentSupportLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
build: PersistentSupportBuild
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublishedCameraEgoMotionLabInstance:
|
|
binding: LabSessionBinding
|
|
job: CameraComputeJob
|
|
result: IntegratedPerceptionResult
|
|
build: CameraEgoMotionBuild
|
|
|
|
|
|
def publish_integrated_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
source_result_root: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
provenance: dict[str, Any] | None = None,
|
|
) -> PublishedIntegratedLabInstance:
|
|
"""Snapshot one accepted visual result as a separately replayable LAB run.
|
|
|
|
Large immutable camera, LiDAR, array and RRD payloads are hard-linked.
|
|
Content-addressed manifests are rebound to the LAB session id, so the
|
|
existing viewer resolves one exact result instead of selecting whichever
|
|
source-session result happens to be newest.
|
|
"""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
source_result_path = source_result_root.expanduser().resolve(strict=True)
|
|
source_document = _read_object(source_result_path / "result.json", source_result_path)
|
|
identity = source_document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError("integrated LAB source has no job identity")
|
|
source_job_root = jobs_root / identity["job_id"]
|
|
source = validate_integrated_perception_result(
|
|
source_job_root,
|
|
source_result_path,
|
|
packs_root,
|
|
)
|
|
if not source.accepted:
|
|
raise SessionIntegrityError("only accepted integrated results can become LAB instances")
|
|
|
|
lab_job = _publish_lab_job(source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(source, lab_job, packs_root, lab_session_id)
|
|
lab_result = _publish_lab_result(
|
|
source,
|
|
lab_job,
|
|
lab_pack,
|
|
results_root,
|
|
lab_session_id,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
lab_result,
|
|
packs_root,
|
|
)
|
|
store = SessionStore(root)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source.job.session_id,
|
|
lab_session_id=lab_session_id,
|
|
)
|
|
configuration = source_document["identity"].get("configuration")
|
|
profile_sha256 = (
|
|
configuration.get("profile_sha256") if isinstance(configuration, dict) else None
|
|
)
|
|
if not isinstance(profile_sha256, str) or len(profile_sha256) != 64:
|
|
profile_sha256 = None
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source.job.session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e10-integrated-perception",
|
|
result_id=validated.result_id,
|
|
source_result_id=source.result_id,
|
|
config_sha256=profile_sha256,
|
|
run_created_at_utc=source.created_at_utc,
|
|
provenance={
|
|
"schema_version": "missioncore.integrated-lab-publication/v1",
|
|
"storage_mode": "hard-linked-immutable-payloads",
|
|
"source_job_id": source.job.job_id,
|
|
"projected_job_id": lab_job.job_id,
|
|
"source_lidar_pack_id": source.pack_root.name,
|
|
"projected_lidar_pack_id": lab_pack.name,
|
|
**(provenance or {}),
|
|
},
|
|
)
|
|
return PublishedIntegratedLabInstance(binding=binding, job=lab_job, result=validated)
|
|
|
|
|
|
def publish_e21_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
e21_result_root: Path,
|
|
worker_result_root: Path,
|
|
semantic_reference_result_root: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedIntegratedLabInstance:
|
|
"""Publish the accepted E21 envelope as an exact 60-second visual LAB run.
|
|
|
|
E21 persisted semantic mask identities but intentionally did not duplicate
|
|
mask pixels. The immutable full-session reference contains those exact
|
|
pixels. Publication therefore materializes a bounded viewer projection only
|
|
after every E21 mask SHA-256 matches the corresponding reference mask.
|
|
Detector latest-wins replacements remain explicit empty frames.
|
|
"""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
reference_path = semantic_reference_result_root.expanduser().resolve(strict=True)
|
|
reference_document = _read_object(reference_path / "result.json", reference_path)
|
|
reference_identity = reference_document.get("identity")
|
|
if not isinstance(reference_identity, dict) or not isinstance(
|
|
reference_identity.get("job_id"), str
|
|
):
|
|
raise SessionIntegrityError("E21 semantic reference has no job identity")
|
|
reference = validate_integrated_perception_result(
|
|
jobs_root / reference_identity["job_id"],
|
|
reference_path,
|
|
packs_root,
|
|
)
|
|
if not reference.accepted or reference.source_start_frame_index != 0:
|
|
raise SessionIntegrityError("E21 semantic reference is not an accepted zero-based run")
|
|
|
|
e21_root = e21_result_root.expanduser().resolve(strict=True)
|
|
worker_root = worker_result_root.expanduser().resolve(strict=True)
|
|
e21_document, e21_report, worker_document = _validate_e21_inputs(
|
|
e21_root,
|
|
worker_root,
|
|
)
|
|
frame_count = int(e21_report["metrics"]["source"]["events_admitted"]["camera-frame"])
|
|
if frame_count < 2 or frame_count > reference.frame_count:
|
|
raise SessionIntegrityError("E21 frame count is outside the semantic reference")
|
|
|
|
lab_job = _publish_lab_job(reference.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_e21_pack(
|
|
reference,
|
|
lab_job,
|
|
packs_root,
|
|
lab_session_id,
|
|
frame_count,
|
|
)
|
|
lab_result = _publish_e21_visual_result(
|
|
reference=reference,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
lab_session_id=lab_session_id,
|
|
frame_count=frame_count,
|
|
e21_root=e21_root,
|
|
e21_document=e21_document,
|
|
e21_report=e21_report,
|
|
worker_root=worker_root,
|
|
worker_document=worker_document,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
lab_result,
|
|
packs_root,
|
|
)
|
|
store = SessionStore(root)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=reference.job.session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(validated.timeline_start_seconds * 1_000_000_000),
|
|
timeline_end_ns=round(validated.timeline_end_seconds * 1_000_000_000),
|
|
)
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=reference.job.session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e21-realtime-envelope",
|
|
result_id=validated.result_id,
|
|
source_result_id=str(e21_document["result_id"]),
|
|
config_sha256=str(e21_report["identity"]["profile_sha256"]),
|
|
run_created_at_utc=str(e21_report["created_at_utc"]),
|
|
duration_seconds=(validated.timeline_end_seconds - validated.timeline_start_seconds),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e21-lab-publication/v1",
|
|
"storage_mode": "bounded-derived-replay-and-projection",
|
|
"e21_result_id": e21_document["result_id"],
|
|
"worker_result_id": worker_document["result_id"],
|
|
"semantic_reference_result_id": reference.result_id,
|
|
"semantic_mask_reconstruction": "exact-sha256-match",
|
|
"detector_replacement_frames": [
|
|
12,
|
|
239,
|
|
240,
|
|
241,
|
|
242,
|
|
433,
|
|
498,
|
|
],
|
|
"visual_frame_count": frame_count,
|
|
"timeline_start_seconds": validated.timeline_start_seconds,
|
|
"timeline_end_seconds": validated.timeline_end_seconds,
|
|
"source_payloads_mutated": False,
|
|
},
|
|
)
|
|
return PublishedIntegratedLabInstance(binding=binding, job=lab_job, result=validated)
|
|
|
|
|
|
def publish_e22_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
source_result_root: Path,
|
|
profile_path: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedTemporalLabInstance:
|
|
"""Derive and publish one bounded temporal-stability LAB comparison."""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
source_path = source_result_root.expanduser().resolve(strict=True)
|
|
source_document = _read_object(source_path / "result.json", source_path)
|
|
identity = source_document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError("E22 source has no job identity")
|
|
source = validate_integrated_perception_result(
|
|
jobs_root / identity["job_id"],
|
|
source_path,
|
|
packs_root,
|
|
)
|
|
if not source.accepted:
|
|
raise SessionIntegrityError("E22 source result is not accepted")
|
|
|
|
lab_job = _publish_lab_job(source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(source, lab_job, packs_root, lab_session_id)
|
|
build = build_temporal_stability_result(
|
|
source=source,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
profile_path=profile_path,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
build.result_root,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted:
|
|
failed = [
|
|
name for name, accepted in build.report["acceptance"]["checks"].items() if not accepted
|
|
]
|
|
raise SessionIntegrityError(f"E22 temporal acceptance failed: {', '.join(failed)}")
|
|
|
|
store = SessionStore(root)
|
|
source_lab = store.get_lab_instance(source.job.session_id)
|
|
source_session_id = (
|
|
source.job.session_id if source_lab is None else source_lab.source_session_id
|
|
)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source_session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(validated.timeline_start_seconds * 1_000_000_000),
|
|
timeline_end_ns=round(validated.timeline_end_seconds * 1_000_000_000),
|
|
)
|
|
metrics = build.report["metrics"]
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source_session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e22-temporal-stability",
|
|
result_id=validated.result_id,
|
|
source_result_id=source.result_id,
|
|
config_sha256=build.profile_sha256,
|
|
run_created_at_utc=validated.created_at_utc,
|
|
duration_seconds=(validated.timeline_end_seconds - validated.timeline_start_seconds),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e22-lab-publication/v1",
|
|
"storage_mode": "bounded-derived-replay-and-temporal-projection",
|
|
"source_result_id": source.result_id,
|
|
"source_lab_session_id": (None if source_lab is None else source_lab.session_id),
|
|
"source_payloads_mutated": False,
|
|
"lookahead_frames": 0,
|
|
"peak_track_states": metrics["runtime"]["peak_track_states"],
|
|
"camera_frame_processing_p95_ms": metrics["runtime"]["camera_frame_processing_ms"][
|
|
"p95"
|
|
],
|
|
"semantic_frame_processing_p95_ms": metrics["runtime"]["semantic_frame_processing_ms"][
|
|
"p95"
|
|
],
|
|
"quality_reductions": metrics["reductions"],
|
|
},
|
|
)
|
|
return PublishedTemporalLabInstance(
|
|
binding=binding,
|
|
job=lab_job,
|
|
result=validated,
|
|
build=build,
|
|
)
|
|
|
|
|
|
def publish_e23_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
reference_result_root: Path,
|
|
worker_result_root: Path,
|
|
source_report_path: Path,
|
|
profile_path: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedIntegratedLabInstance:
|
|
"""Publish one accepted inline-temporal 1x worker run as an exact LAB replay."""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
reference_path = reference_result_root.expanduser().resolve(strict=True)
|
|
reference_document = _read_object(reference_path / "result.json", reference_path)
|
|
reference_identity = reference_document.get("identity")
|
|
if not isinstance(reference_identity, dict) or not isinstance(
|
|
reference_identity.get("job_id"), str
|
|
):
|
|
raise SessionIntegrityError("E23 semantic reference has no job identity")
|
|
reference = validate_integrated_perception_result(
|
|
jobs_root / reference_identity["job_id"],
|
|
reference_path,
|
|
packs_root,
|
|
)
|
|
if not reference.accepted or reference.source_start_frame_index != 0:
|
|
raise SessionIntegrityError("E23 reference is not an accepted zero-based run")
|
|
|
|
profile, profile_sha256 = read_inline_profile(profile_path)
|
|
worker_root = worker_result_root.expanduser().resolve(strict=True)
|
|
source_path = source_report_path.expanduser().resolve(strict=True)
|
|
worker_document, worker_report, source_report = _validate_e23_inputs(
|
|
worker_root,
|
|
source_path,
|
|
profile_sha256,
|
|
)
|
|
frame_count = int(source_report["events_selected"]["camera-frame"])
|
|
if frame_count != reference.frame_count:
|
|
raise SessionIntegrityError("E23 source and reference frame counts differ")
|
|
|
|
lab_job = _publish_lab_job(reference.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_e21_pack(
|
|
reference,
|
|
lab_job,
|
|
packs_root,
|
|
lab_session_id,
|
|
frame_count,
|
|
visual_projection="accepted-e23-inline-envelope/v1",
|
|
)
|
|
lab_result, quality = _publish_e23_visual_result(
|
|
reference=reference,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
lab_session_id=lab_session_id,
|
|
frame_count=frame_count,
|
|
worker_root=worker_root,
|
|
worker_document=worker_document,
|
|
worker_report=worker_report,
|
|
source_report=source_report,
|
|
profile=profile,
|
|
profile_sha256=profile_sha256,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
lab_result,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted or not all(quality["checks"].values()):
|
|
failed = [name for name, accepted in quality["checks"].items() if not accepted]
|
|
raise SessionIntegrityError(f"E23 inline temporal acceptance failed: {', '.join(failed)}")
|
|
|
|
store = SessionStore(root)
|
|
source_lab = store.get_lab_instance(reference.job.session_id)
|
|
source_session_id = (
|
|
reference.job.session_id if source_lab is None else source_lab.source_session_id
|
|
)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source_session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(validated.timeline_start_seconds * 1_000_000_000),
|
|
timeline_end_ns=round(validated.timeline_end_seconds * 1_000_000_000),
|
|
)
|
|
temporal = worker_report["metrics"]["temporal_stability"]
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source_session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e23-inline-temporal-stability",
|
|
result_id=validated.result_id,
|
|
source_result_id=str(worker_document["result_id"]),
|
|
config_sha256=profile_sha256,
|
|
run_created_at_utc=str(worker_report["created_at_utc"]),
|
|
duration_seconds=(validated.timeline_end_seconds - validated.timeline_start_seconds),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e23-lab-publication/v1",
|
|
"storage_mode": "bounded-inline-worker-result-and-immutable-source-replay",
|
|
"worker_result_id": worker_document["result_id"],
|
|
"source_report_sha256": _sha256(source_path),
|
|
"reference_result_id": reference.result_id,
|
|
"source_payloads_mutated": False,
|
|
"lookahead_frames": 0,
|
|
"speed": 1.0,
|
|
"quality_reductions": quality["reductions"],
|
|
"temporal_2d_3d_p95_ms": worker_report["metrics"]["latency_ms"]["temporal_2d_3d_ms"][
|
|
"p95"
|
|
],
|
|
"semantic_temporal_p95_ms": temporal["semantic"]["processing_ms"]["p95"],
|
|
"peak_track_states": temporal["tracking_2d_3d"]["peak_track_states"],
|
|
"rss_growth_mib": worker_report["metrics"]["runtime_telemetry"]["rss_growth_mib"],
|
|
},
|
|
)
|
|
return PublishedIntegratedLabInstance(
|
|
binding=binding,
|
|
job=lab_job,
|
|
result=validated,
|
|
)
|
|
|
|
|
|
def publish_e24_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
source_result_root: Path,
|
|
profile_path: Path,
|
|
benchmark_path: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedWorldMotionLabInstance:
|
|
"""Derive and publish a bounded world-frame motion-tracking LAB run."""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
source_path = source_result_root.expanduser().resolve(strict=True)
|
|
source_document = _read_object(source_path / "result.json", source_path)
|
|
identity = source_document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError("E24 source has no job identity")
|
|
source = validate_integrated_perception_result(
|
|
jobs_root / identity["job_id"],
|
|
source_path,
|
|
packs_root,
|
|
)
|
|
if not source.accepted:
|
|
raise SessionIntegrityError("E24 source result is not accepted")
|
|
|
|
lab_job = _publish_lab_job(source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(source, lab_job, packs_root, lab_session_id)
|
|
build = build_world_motion_result(
|
|
source=source,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
profile_path=profile_path,
|
|
benchmark_path=benchmark_path,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
build.result_root,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted:
|
|
failed = [
|
|
name for name, accepted in build.report["acceptance"]["checks"].items() if not accepted
|
|
]
|
|
raise SessionIntegrityError(
|
|
f"E24 world-motion artifact acceptance failed: {', '.join(failed)}"
|
|
)
|
|
|
|
store = SessionStore(root)
|
|
source_lab = store.get_lab_instance(source.job.session_id)
|
|
source_session_id = (
|
|
source.job.session_id if source_lab is None else source_lab.source_session_id
|
|
)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source_session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(validated.timeline_start_seconds * 1_000_000_000),
|
|
timeline_end_ns=round(validated.timeline_end_seconds * 1_000_000_000),
|
|
)
|
|
metrics = build.report["metrics"]
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source_session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e24-world-motion",
|
|
result_id=validated.result_id,
|
|
source_result_id=source.result_id,
|
|
config_sha256=build.profile_sha256,
|
|
run_created_at_utc=validated.created_at_utc,
|
|
duration_seconds=(validated.timeline_end_seconds - validated.timeline_start_seconds),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e24-lab-publication/v1",
|
|
"storage_mode": "bounded-world-frame-tracking-and-immutable-source-replay",
|
|
"source_result_id": source.result_id,
|
|
"source_lab_session_id": (None if source_lab is None else source_lab.session_id),
|
|
"source_payloads_mutated": False,
|
|
"coordinate_frame": "k1-map",
|
|
"lookahead_frames": 0,
|
|
"benchmark_sha256": build.benchmark_sha256,
|
|
"benchmark_passed": metrics["benchmark"]["passed"],
|
|
"benchmark_passed_events": metrics["benchmark"]["passed_events"],
|
|
"benchmark_total_events": metrics["benchmark"]["total_events"],
|
|
"world_motion_processing_p95_ms": metrics["runtime"][
|
|
"world_motion_frame_processing_ms"
|
|
]["p95"],
|
|
"peak_tracks": metrics["runtime"]["peak_tracks"],
|
|
"navigation_or_safety_accepted": False,
|
|
},
|
|
)
|
|
return PublishedWorldMotionLabInstance(
|
|
binding=binding,
|
|
job=lab_job,
|
|
result=validated,
|
|
build=build,
|
|
)
|
|
|
|
|
|
def publish_e25_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
source_result_root: Path,
|
|
world_motion_profile_path: Path,
|
|
profile_path: Path,
|
|
benchmark_path: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedPersistentSupportLabInstance:
|
|
"""Derive and publish one bounded persistent-support motion LAB run."""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
source_path = source_result_root.expanduser().resolve(strict=True)
|
|
source_document = _read_object(source_path / "result.json", source_path)
|
|
identity = source_document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError("E25 source has no job identity")
|
|
source = validate_integrated_perception_result(
|
|
jobs_root / identity["job_id"],
|
|
source_path,
|
|
packs_root,
|
|
)
|
|
if not source.accepted:
|
|
raise SessionIntegrityError("E25 source result is not accepted")
|
|
|
|
lab_job = _publish_lab_job(source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(source, lab_job, packs_root, lab_session_id)
|
|
build = build_persistent_support_result(
|
|
source=source,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
world_motion_profile_path=world_motion_profile_path,
|
|
profile_path=profile_path,
|
|
benchmark_path=benchmark_path,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
build.result_root,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted:
|
|
failed = [
|
|
name for name, accepted in build.report["acceptance"]["checks"].items() if not accepted
|
|
]
|
|
raise SessionIntegrityError(
|
|
f"E25 persistent-support artifact acceptance failed: {', '.join(failed)}"
|
|
)
|
|
|
|
store = SessionStore(root)
|
|
source_lab = store.get_lab_instance(source.job.session_id)
|
|
source_session_id = (
|
|
source.job.session_id if source_lab is None else source_lab.source_session_id
|
|
)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source_session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(validated.timeline_start_seconds * 1_000_000_000),
|
|
timeline_end_ns=round(validated.timeline_end_seconds * 1_000_000_000),
|
|
)
|
|
metrics = build.report["metrics"]
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source_session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e25-persistent-support-motion",
|
|
result_id=validated.result_id,
|
|
source_result_id=source.result_id,
|
|
config_sha256=build.profile_sha256,
|
|
run_created_at_utc=validated.created_at_utc,
|
|
duration_seconds=(validated.timeline_end_seconds - validated.timeline_start_seconds),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e25-lab-publication/v1",
|
|
"storage_mode": "bounded-persistent-support-and-immutable-source-replay",
|
|
"source_result_id": source.result_id,
|
|
"source_lab_session_id": (None if source_lab is None else source_lab.session_id),
|
|
"source_payloads_mutated": False,
|
|
"coordinate_frame": "k1-map",
|
|
"measurement": "persistent-object-support-occupancy",
|
|
"lookahead_frames": 0,
|
|
"benchmark_sha256": build.benchmark_sha256,
|
|
"benchmark_passed": metrics["benchmark"]["passed"],
|
|
"benchmark_passed_events": metrics["benchmark"]["passed_events"],
|
|
"benchmark_total_events": metrics["benchmark"]["total_events"],
|
|
"persistent_support_processing_p95_ms": metrics["runtime"][
|
|
"persistent_support_frame_processing_ms"
|
|
]["p95"],
|
|
"peak_tracks": metrics["runtime"]["peak_tracks"],
|
|
"navigation_or_safety_accepted": False,
|
|
},
|
|
)
|
|
return PublishedPersistentSupportLabInstance(
|
|
binding=binding,
|
|
job=lab_job,
|
|
result=validated,
|
|
build=build,
|
|
)
|
|
|
|
|
|
def publish_e26_lab_instance(
|
|
*,
|
|
repository_root: Path,
|
|
lidar_result_root: Path,
|
|
camera_result_root: Path,
|
|
profile_path: Path,
|
|
benchmark_path: Path,
|
|
lab_session_id: str,
|
|
lab_id: str,
|
|
display_name: str,
|
|
) -> PublishedCameraEgoMotionLabInstance:
|
|
"""Derive and publish one bounded camera/ego-motion evidence LAB run."""
|
|
|
|
root = repository_root.expanduser().resolve(strict=True)
|
|
jobs_root = root / ".runtime" / "compute-jobs"
|
|
results_root = root / ".runtime" / "compute-experiments" / "e10" / "worker-results"
|
|
packs_root = root / ".runtime" / "compute-experiments" / "e10" / "lidar-packs"
|
|
|
|
def source(path: Path, label: str) -> IntegratedPerceptionResult:
|
|
resolved = path.expanduser().resolve(strict=True)
|
|
document = _read_object(resolved / "result.json", resolved)
|
|
identity = document.get("identity")
|
|
if not isinstance(identity, dict) or not isinstance(identity.get("job_id"), str):
|
|
raise SessionIntegrityError(f"E26 {label} source has no job identity")
|
|
validated = validate_integrated_perception_result(
|
|
jobs_root / identity["job_id"],
|
|
resolved,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted:
|
|
raise SessionIntegrityError(f"E26 {label} source result is not accepted")
|
|
return validated
|
|
|
|
lidar_source = source(lidar_result_root, "LiDAR")
|
|
camera_source = source(camera_result_root, "camera")
|
|
if lidar_source.job.source_id != camera_source.job.source_id:
|
|
raise SessionIntegrityError("E26 source camera identities differ")
|
|
|
|
lab_job = _publish_lab_job(lidar_source.job, jobs_root, lab_session_id)
|
|
lab_pack = _publish_lab_pack(
|
|
lidar_source,
|
|
lab_job,
|
|
packs_root,
|
|
lab_session_id,
|
|
)
|
|
build = build_camera_ego_motion_result(
|
|
lidar_source=lidar_source,
|
|
camera_source=camera_source,
|
|
lab_job=lab_job,
|
|
lab_pack=lab_pack,
|
|
results_root=results_root,
|
|
profile_path=profile_path,
|
|
benchmark_path=benchmark_path,
|
|
)
|
|
validated = validate_integrated_perception_result(
|
|
lab_job.job_root,
|
|
build.result_root,
|
|
packs_root,
|
|
)
|
|
if not validated.accepted:
|
|
failed = [
|
|
name
|
|
for name, accepted in build.report["acceptance"]["checks"].items()
|
|
if not accepted
|
|
]
|
|
raise SessionIntegrityError(
|
|
f"E26 camera/ego-motion artifact acceptance failed: {', '.join(failed)}"
|
|
)
|
|
|
|
store = SessionStore(root)
|
|
source_lab = store.get_lab_instance(lidar_source.job.session_id)
|
|
source_session_id = (
|
|
lidar_source.job.session_id
|
|
if source_lab is None
|
|
else source_lab.source_session_id
|
|
)
|
|
publish_lab_replay_cache(
|
|
store.data_dir,
|
|
source_session_id=source_session_id,
|
|
lab_session_id=lab_session_id,
|
|
timeline_start_ns=round(
|
|
validated.timeline_start_seconds * 1_000_000_000
|
|
),
|
|
timeline_end_ns=round(
|
|
validated.timeline_end_seconds * 1_000_000_000
|
|
),
|
|
)
|
|
metrics = build.report["metrics"]
|
|
binding = store.publish_lab_instance(
|
|
session_id=lab_session_id,
|
|
source_session_id=source_session_id,
|
|
display_name=display_name,
|
|
lab_id=lab_id,
|
|
result_kind="e26-camera-ego-motion-fusion",
|
|
result_id=validated.result_id,
|
|
source_result_id=lidar_source.result_id,
|
|
config_sha256=build.profile_sha256,
|
|
run_created_at_utc=validated.created_at_utc,
|
|
duration_seconds=(
|
|
validated.timeline_end_seconds
|
|
- validated.timeline_start_seconds
|
|
),
|
|
include_recorded_media=False,
|
|
provenance={
|
|
"schema_version": "missioncore.e26-lab-publication/v1",
|
|
"storage_mode": (
|
|
"bounded-camera-ego-motion-and-immutable-lidar-source-replay"
|
|
),
|
|
"source_result_id": lidar_source.result_id,
|
|
"camera_source_result_id": camera_source.result_id,
|
|
"source_lab_session_id": (
|
|
None if source_lab is None else source_lab.session_id
|
|
),
|
|
"source_payloads_mutated": False,
|
|
"coordinate_frame": "k1-map",
|
|
"camera_measurement": "kb4-multiview-static-world-hypothesis",
|
|
"metric_measurement": "e25-persistent-lidar-support",
|
|
"lookahead_frames": 0,
|
|
"benchmark_sha256": build.benchmark_sha256,
|
|
"benchmark_passed": metrics["benchmark"]["passed"],
|
|
"benchmark_passed_events": metrics["benchmark"]["passed_events"],
|
|
"benchmark_total_events": metrics["benchmark"]["total_events"],
|
|
"camera_ego_motion_processing_p95_ms": metrics["runtime"][
|
|
"camera_ego_motion_frame_processing_ms"
|
|
]["p95"],
|
|
"peak_tracks": metrics["runtime"]["peak_tracks"],
|
|
"camera_only_metric_velocity_valid": False,
|
|
"navigation_or_safety_accepted": False,
|
|
},
|
|
)
|
|
return PublishedCameraEgoMotionLabInstance(
|
|
binding=binding,
|
|
job=lab_job,
|
|
result=validated,
|
|
build=build,
|
|
)
|
|
|
|
|
|
def _validate_e23_inputs(
|
|
worker_root: Path,
|
|
source_report_path: Path,
|
|
profile_sha256: str,
|
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
worker_document = _read_object(worker_root / "result.json", worker_root)
|
|
worker_report = _read_object(worker_root / "run-report.json", worker_root)
|
|
source_report = _read_object(source_report_path, source_report_path.parent)
|
|
worker_identity = worker_document.get("identity")
|
|
report_identity = worker_report.get("identity")
|
|
if (
|
|
worker_document.get("schema_version") != "missioncore.e15-shadow-inference-result/v1"
|
|
or worker_document.get("result_id") != worker_root.name
|
|
or worker_document.get("acceptance_state") != "accepted"
|
|
or worker_document.get("publication_scope") != "live-shadow-diagnostic-only"
|
|
or not isinstance(worker_identity, dict)
|
|
or worker_identity.get("pipeline")
|
|
!= "warm-worker-inline-bounded-temporal-2d-3d-semantic/v1"
|
|
or worker_identity.get("profiles", {}).get("stability_sha256") != profile_sha256
|
|
or worker_report.get("schema_version") != "missioncore.e15-shadow-inference-report/v1"
|
|
or worker_report.get("result_id") != worker_root.name
|
|
or worker_report.get("state") != "accepted"
|
|
or report_identity != worker_identity
|
|
or not all(worker_report.get("acceptance", {}).get("checks", {}).values())
|
|
or source_report.get("schema_version") != "missioncore.e23-replay-source-report/v1"
|
|
or source_report.get("state") != "completed"
|
|
or source_report.get("session_id") != worker_identity.get("session_id")
|
|
or source_report.get("source", {}).get("speed") != 1.0
|
|
or source_report.get("authority", {}).get("mode") != "shadow-diagnostic-only"
|
|
or source_report.get("authority", {}).get("commands_enabled") is not False
|
|
or source_report.get("authority", {}).get("navigation_or_safety_accepted") is not False
|
|
):
|
|
raise SessionIntegrityError("E23 accepted worker/source identity is inconsistent")
|
|
selected = source_report.get("events_selected")
|
|
diagnostics = source_report.get("diagnostic_results")
|
|
if (
|
|
not isinstance(selected, dict)
|
|
or selected.get("camera-frame") != 601
|
|
or selected.get("lidar") != 585
|
|
or selected.get("pose") != 600
|
|
or not isinstance(diagnostics, dict)
|
|
or int(diagnostics.get("received", 0)) < 590
|
|
):
|
|
raise SessionIntegrityError("E23 source replay coverage is incomplete")
|
|
|
|
required = {
|
|
"e15-semantic-frames": "semantic-frames.jsonl",
|
|
"e23-raw-fusion-frames": "raw-fusion-frames.jsonl",
|
|
"e15-fusion-frames": "fusion-frames.jsonl",
|
|
"e15-world-state": "world-state.jsonl",
|
|
"worker-gpu-telemetry": "gpu-telemetry.jsonl",
|
|
"worker-runtime-telemetry": "runtime-telemetry.jsonl",
|
|
"e15-run-report": "run-report.json",
|
|
}
|
|
artifacts = worker_document.get("artifacts")
|
|
descriptors = (
|
|
{
|
|
value.get("kind"): value
|
|
for value in artifacts
|
|
if isinstance(value, dict) and value.get("kind") in required
|
|
}
|
|
if isinstance(artifacts, list)
|
|
else {}
|
|
)
|
|
if set(descriptors) != set(required):
|
|
raise SessionIntegrityError("E23 worker artifacts are incomplete")
|
|
for kind, name in required.items():
|
|
descriptor = descriptors[kind]
|
|
path = worker_root / name
|
|
if (
|
|
descriptor.get("path") != name
|
|
or descriptor.get("byte_length") != path.stat().st_size
|
|
or descriptor.get("sha256") != _sha256(path)
|
|
):
|
|
raise SessionIntegrityError("E23 worker artifact identity changed")
|
|
return worker_document, worker_report, source_report
|
|
|
|
|
|
def _validate_e21_inputs(
|
|
e21_root: Path,
|
|
worker_root: Path,
|
|
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
e21_document = _read_object(e21_root / "result.json", e21_root)
|
|
e21_report = _read_object(e21_root / "report.json", e21_root)
|
|
report_descriptor = e21_document.get("report")
|
|
if (
|
|
e21_document.get("schema_version") != "missioncore.e21-realtime-envelope-result/v1"
|
|
or e21_document.get("result_id") != e21_root.name
|
|
or e21_document.get("state") != "accepted"
|
|
or not isinstance(report_descriptor, dict)
|
|
or report_descriptor.get("path") != "report.json"
|
|
or report_descriptor.get("byte_length") != (e21_root / "report.json").stat().st_size
|
|
or report_descriptor.get("sha256") != _sha256(e21_root / "report.json")
|
|
or e21_report.get("schema_version") != "missioncore.e21-realtime-envelope-report/v1"
|
|
or e21_report.get("result_id") != e21_root.name
|
|
or e21_report.get("state") != "accepted"
|
|
or not isinstance(e21_report.get("identity"), dict)
|
|
or not isinstance(e21_report.get("metrics"), dict)
|
|
):
|
|
raise SessionIntegrityError("E21 accepted result identity is inconsistent")
|
|
|
|
worker_document = _read_object(worker_root / "result.json", worker_root)
|
|
e21_identity = e21_report["identity"]
|
|
if (
|
|
worker_document.get("schema_version") != "missioncore.e15-shadow-inference-result/v1"
|
|
or worker_document.get("result_id") != worker_root.name
|
|
or worker_document.get("result_id") != e21_identity.get("worker_result_id")
|
|
or _sha256(worker_root / "result.json") != e21_identity.get("worker_result_sha256")
|
|
or not isinstance(worker_document.get("identity"), dict)
|
|
or not isinstance(worker_document.get("artifacts"), list)
|
|
):
|
|
raise SessionIntegrityError("E21 worker result identity is inconsistent")
|
|
required = {
|
|
"e15-semantic-frames": "semantic-frames.jsonl",
|
|
"e15-fusion-frames": "fusion-frames.jsonl",
|
|
"e15-world-state": "world-state.jsonl",
|
|
"worker-gpu-telemetry": "gpu-telemetry.jsonl",
|
|
}
|
|
descriptors = {
|
|
value.get("kind"): value
|
|
for value in worker_document["artifacts"]
|
|
if isinstance(value, dict) and value.get("kind") in required
|
|
}
|
|
if set(descriptors) != set(required):
|
|
raise SessionIntegrityError("E21 worker artifacts are incomplete")
|
|
for kind, name in required.items():
|
|
descriptor = descriptors[kind]
|
|
path = worker_root / name
|
|
if (
|
|
descriptor.get("path") != name
|
|
or descriptor.get("byte_length") != path.stat().st_size
|
|
or descriptor.get("sha256") != _sha256(path)
|
|
):
|
|
raise SessionIntegrityError("E21 worker artifact identity changed")
|
|
return e21_document, e21_report, worker_document
|
|
|
|
|
|
def _publish_e21_pack(
|
|
reference: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
packs_root: Path,
|
|
lab_session_id: str,
|
|
frame_count: int,
|
|
*,
|
|
visual_projection: str = "accepted-e21-envelope/v1",
|
|
) -> Path:
|
|
source_manifest = _read_object(reference.pack_root / "manifest.json", reference.pack_root)
|
|
with np.load(reference.pack_root / "lidar-pack.npz", allow_pickle=False) as arrays:
|
|
cloud_end = int(arrays["cloud_offsets"][frame_count])
|
|
payload = {
|
|
"frame_indices": arrays["frame_indices"][:frame_count].copy(),
|
|
"source_frame_indices": arrays["source_frame_indices"][:frame_count].copy(),
|
|
"session_seconds": arrays["session_seconds"][:frame_count].copy(),
|
|
"sample_available": arrays["sample_available"][:frame_count].copy(),
|
|
"cloud_offsets": arrays["cloud_offsets"][: frame_count + 1].copy(),
|
|
"cloud_points_map": arrays["cloud_points_map"][:cloud_end].copy(),
|
|
"pose_positions_map": arrays["pose_positions_map"][:frame_count].copy(),
|
|
"pose_quaternions_map_from_lidar": arrays["pose_quaternions_map_from_lidar"][
|
|
:frame_count
|
|
].copy(),
|
|
"lidar_camera_delta_ms": arrays["lidar_camera_delta_ms"][:frame_count].copy(),
|
|
"pose_point_delta_ms": arrays["pose_point_delta_ms"][:frame_count].copy(),
|
|
"intrinsic_fx_fy_cx_cy": arrays["intrinsic_fx_fy_cx_cy"].copy(),
|
|
"distortion_kb4": arrays["distortion_kb4"].copy(),
|
|
"t_camera_from_lidar": arrays["t_camera_from_lidar"].copy(),
|
|
}
|
|
identity = json.loads(json.dumps(source_manifest["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"frame_count": frame_count,
|
|
"source_start_frame_index": 0,
|
|
"source_end_frame_index": frame_count - 1,
|
|
"available_lidar_frames": int(np.count_nonzero(payload["sample_available"])),
|
|
"point_count": int(payload["cloud_points_map"].shape[0]),
|
|
"timeline_start_seconds": float(payload["session_seconds"][0]),
|
|
"timeline_end_seconds": float(payload["session_seconds"][-1]),
|
|
"visual_projection": visual_projection,
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
pack_id = f"e10-lidar-pack-{identity_sha256}"
|
|
destination = packs_root / pack_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "manifest.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("E21 LAB LiDAR pack id collides")
|
|
return destination
|
|
staging = _staging_directory(packs_root, pack_id)
|
|
try:
|
|
arrays_path = staging / "lidar-pack.npz"
|
|
np.savez_compressed(arrays_path, **payload)
|
|
os.chmod(arrays_path, 0o600)
|
|
write_json_atomic(
|
|
staging / "manifest.json",
|
|
{
|
|
**source_manifest,
|
|
"pack_id": pack_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": str(source_manifest["created_at_utc"]),
|
|
"artifact": {
|
|
"path": arrays_path.name,
|
|
"media_type": "application/x-npz",
|
|
"byte_length": arrays_path.stat().st_size,
|
|
"sha256": _sha256(arrays_path),
|
|
},
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _publish_e21_visual_result(
|
|
*,
|
|
reference: IntegratedPerceptionResult,
|
|
lab_job: CameraComputeJob,
|
|
lab_pack: Path,
|
|
results_root: Path,
|
|
lab_session_id: str,
|
|
frame_count: int,
|
|
e21_root: Path,
|
|
e21_document: dict[str, Any],
|
|
e21_report: dict[str, Any],
|
|
worker_root: Path,
|
|
worker_document: dict[str, Any],
|
|
) -> Path:
|
|
with np.load(reference.arrays_path, allow_pickle=False) as arrays:
|
|
frame_times = arrays["frame_times_ns"][:frame_count].copy()
|
|
reference_semantic_indices = arrays["semantic_frame_indices"]
|
|
selected = reference_semantic_indices < frame_count
|
|
reference_indices = reference_semantic_indices[selected].copy()
|
|
reference_masks = arrays["semantic_masks"][selected].copy()
|
|
|
|
semantic_rows = _read_jsonl(worker_root / "semantic-frames.jsonl")
|
|
if [row.get("frame_index") for row in semantic_rows] != reference_indices.tolist():
|
|
raise SessionIntegrityError("E21 semantic frame schedule changed")
|
|
for row, mask in zip(semantic_rows, reference_masks, strict=True):
|
|
if row.get("mask_sha256") != hashlib.sha256(mask.tobytes()).hexdigest():
|
|
raise SessionIntegrityError("E21 semantic mask does not match immutable reference")
|
|
row["schema_version"] = "missioncore.e10-semantic-frame/v1"
|
|
row["session_seconds"] = float(frame_times[int(row["frame_index"])]) / 1_000_000_000
|
|
|
|
fusion_source = {
|
|
int(row["source_frame_index"]): row
|
|
for row in _read_jsonl(worker_root / "fusion-frames.jsonl")
|
|
}
|
|
world_source = {
|
|
int(row["source_frame_index"]): row
|
|
for row in _read_jsonl(worker_root / "world-state.jsonl")
|
|
}
|
|
if set(fusion_source) != set(world_source):
|
|
raise SessionIntegrityError("E21 fusion and world timelines differ")
|
|
fusion_rows: list[dict[str, Any]] = []
|
|
world_rows: list[dict[str, Any]] = []
|
|
dropped_indices: list[int] = []
|
|
for index in range(frame_count):
|
|
session_seconds = float(frame_times[index]) / 1_000_000_000
|
|
fusion = fusion_source.get(index)
|
|
world = world_source.get(index)
|
|
if fusion is None or world is None:
|
|
dropped_indices.append(index)
|
|
fusion_rows.append(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
"fusion_state": "detector-dropped-latest-wins",
|
|
"semantic_source_frame_index": None,
|
|
"semantic_status": "unavailable",
|
|
"objects": [],
|
|
}
|
|
)
|
|
world_rows.append(_dropped_world_row(index, session_seconds))
|
|
continue
|
|
normalized_fusion = json.loads(json.dumps(fusion))
|
|
normalized_fusion.update(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
normalized_world = json.loads(json.dumps(world))
|
|
normalized_world.update(
|
|
{
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
fusion_rows.append(normalized_fusion)
|
|
world_rows.append(normalized_world)
|
|
expected_drops = int(e21_report["metrics"]["worker"]["detector"]["queue"]["dropped_overflow"])
|
|
if len(dropped_indices) != expected_drops:
|
|
raise SessionIntegrityError("E21 detector replacement accounting changed")
|
|
|
|
box_offsets = [0]
|
|
centers: list[list[float]] = []
|
|
half_sizes: list[list[float]] = []
|
|
quaternions: list[list[float]] = []
|
|
colors: list[list[int]] = []
|
|
for row in fusion_rows:
|
|
for item in row["objects"]:
|
|
if not str(item.get("cuboid_status", "")).startswith("accepted-"):
|
|
continue
|
|
centers.append(item["cuboid_center_map"])
|
|
half_sizes.append(item["cuboid_half_size"])
|
|
quaternions.append(item["cuboid_quaternion_xyzw"])
|
|
colors.append(_cuboid_color(item))
|
|
box_offsets.append(len(centers))
|
|
|
|
reference_identity = _read_object(
|
|
reference.result_root / "result.json",
|
|
reference.result_root,
|
|
)["identity"]
|
|
worker_identity = worker_document["identity"]
|
|
configuration = {
|
|
"pipeline": "e21-shadow-realtime-envelope-visual-projection/v1",
|
|
"profile_sha256": e21_report["identity"]["profile_sha256"],
|
|
"profile": {
|
|
"source": reference_identity["configuration"]["profile"]["source"],
|
|
"replay": {
|
|
"speed": 1.0,
|
|
"bounded_latest_wins": True,
|
|
"detector_replacement_frames": dropped_indices,
|
|
},
|
|
},
|
|
"e21_result_id": e21_document["result_id"],
|
|
"worker_result_id": worker_document["result_id"],
|
|
"semantic_mask_materialization": {
|
|
"mode": "immutable-reference-exact-sha256",
|
|
"reference_result_id": reference.result_id,
|
|
"matched_masks": len(semantic_rows),
|
|
},
|
|
}
|
|
selection = {
|
|
"frame_count": frame_count,
|
|
"source_start_frame_index": 0,
|
|
"source_end_frame_index": frame_count - 1,
|
|
"timeline_start_seconds": float(frame_times[0]) / 1_000_000_000,
|
|
"timeline_end_seconds": float(frame_times[-1]) / 1_000_000_000,
|
|
"timeline_sha256": hashlib.sha256(frame_times.tobytes()).hexdigest(),
|
|
}
|
|
identity = {
|
|
"schema_version": "missioncore.e10-integrated-perception-identity/v1",
|
|
"job_id": lab_job.job_id,
|
|
"input_sha256": lab_job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"source_id": lab_job.source_id,
|
|
"lidar_pack_id": lab_pack.name,
|
|
"selection": selection,
|
|
"configuration": configuration,
|
|
"models": worker_identity["models"],
|
|
}
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
result_id = f"e10-integrated-perception-{identity_sha256}"
|
|
destination = results_root / result_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "result.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("E21 LAB visual result id collides")
|
|
return destination
|
|
|
|
staging = _staging_directory(results_root, result_id)
|
|
try:
|
|
semantic_path = staging / "semantic-frames.jsonl"
|
|
fusion_path = staging / "fusion-frames.jsonl"
|
|
world_path = staging / "world-state.jsonl"
|
|
arrays_path = staging / "transient-perception.npz"
|
|
gpu_path = staging / "gpu-telemetry.jsonl"
|
|
report_path = staging / "run-report.json"
|
|
_write_jsonl(semantic_path, semantic_rows)
|
|
_write_jsonl(fusion_path, fusion_rows)
|
|
_write_jsonl(world_path, world_rows)
|
|
np.savez_compressed(
|
|
arrays_path,
|
|
frame_times_ns=frame_times.astype(np.int64, copy=False),
|
|
semantic_frame_indices=reference_indices.astype(np.int64, copy=False),
|
|
semantic_masks=reference_masks.astype(np.uint8, copy=False),
|
|
support_offsets=np.zeros(frame_count + 1, dtype=np.int64),
|
|
support_points=np.empty((0, 3), dtype=np.float32),
|
|
support_colors=np.empty((0, 3), dtype=np.uint8),
|
|
box_offsets=np.asarray(box_offsets, dtype=np.int64),
|
|
box_centers=np.asarray(centers, dtype=np.float32).reshape((-1, 3)),
|
|
box_half_sizes=np.asarray(half_sizes, dtype=np.float32).reshape((-1, 3)),
|
|
box_quaternions=np.asarray(quaternions, dtype=np.float32).reshape((-1, 4)),
|
|
box_colors=np.asarray(colors, dtype=np.uint8).reshape((-1, 4)),
|
|
)
|
|
shutil.copyfile(worker_root / "gpu-telemetry.jsonl", gpu_path)
|
|
report = {
|
|
"schema_version": "missioncore.e10-integrated-perception-report/v1",
|
|
"result_id": result_id,
|
|
"created_at_utc": e21_report["created_at_utc"],
|
|
"state": "accepted",
|
|
"ground_truth": False,
|
|
"identity": identity,
|
|
"acceptance": {
|
|
"accepted": True,
|
|
"navigation_or_safety_accepted": False,
|
|
"checks": {
|
|
"e21_envelope_accepted": True,
|
|
"semantic_hashes_exact": True,
|
|
"detector_replacements_explicit": True,
|
|
"source_payloads_immutable": True,
|
|
},
|
|
},
|
|
"metrics": {
|
|
**e21_report["metrics"],
|
|
"visual_projection": {
|
|
"frames": frame_count,
|
|
"semantic_masks": len(semantic_rows),
|
|
"detector_replacement_frames": dropped_indices,
|
|
"accepted_cuboids": len(centers),
|
|
},
|
|
},
|
|
"runtime": _read_object(worker_root / "run-report.json", worker_root).get(
|
|
"runtime", {}
|
|
),
|
|
"limitations": [
|
|
"This is the exact accepted recorded E21 envelope, not a physical live K1 gate.",
|
|
"Seven latest-wins detector replacements are explicit empty visual frames.",
|
|
"Semantic pixels are materialized only after exact SHA-256 reference matches.",
|
|
"LiDAR support points remain in the immutable source scene and are not duplicated.",
|
|
"Navigation and safety authority remain disabled.",
|
|
],
|
|
}
|
|
write_json_atomic(report_path, report)
|
|
artifacts = [
|
|
_artifact_descriptor(
|
|
"e10-semantic-frames",
|
|
semantic_path,
|
|
"missioncore.e10-semantic-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-fusion-frames",
|
|
fusion_path,
|
|
"missioncore.e10-fusion-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-world-state",
|
|
world_path,
|
|
"missioncore.live-perception-world-state/v1",
|
|
),
|
|
_artifact_descriptor("e10-transient-perception", arrays_path, None),
|
|
_artifact_descriptor("worker-gpu-telemetry", gpu_path, None),
|
|
_artifact_descriptor(
|
|
"e10-run-report",
|
|
report_path,
|
|
"missioncore.e10-integrated-perception-report/v1",
|
|
),
|
|
]
|
|
write_json_atomic(
|
|
staging / "result.json",
|
|
{
|
|
"schema_version": "missioncore.e10-integrated-perception-result/v1",
|
|
"result_id": result_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": e21_report["created_at_utc"],
|
|
"ground_truth": False,
|
|
"publication_scope": "recorded-integrated-realtime-qualification-only",
|
|
"acceptance_state": "accepted",
|
|
"frames_processed": frame_count,
|
|
"artifacts": artifacts,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _publish_e23_visual_result(
|
|
*,
|
|
reference: IntegratedPerceptionResult,
|
|
lab_job: CameraComputeJob,
|
|
lab_pack: Path,
|
|
results_root: Path,
|
|
lab_session_id: str,
|
|
frame_count: int,
|
|
worker_root: Path,
|
|
worker_document: dict[str, Any],
|
|
worker_report: dict[str, Any],
|
|
source_report: dict[str, Any],
|
|
profile: dict[str, Any],
|
|
profile_sha256: str,
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
with np.load(reference.arrays_path, allow_pickle=False) as arrays:
|
|
frame_times = arrays["frame_times_ns"][:frame_count].copy()
|
|
reference_semantic_indices = arrays["semantic_frame_indices"]
|
|
selected = reference_semantic_indices < frame_count
|
|
reference_indices = reference_semantic_indices[selected].copy()
|
|
reference_masks = arrays["semantic_masks"][selected].copy()
|
|
|
|
semantic_rows = _read_jsonl(worker_root / "semantic-frames.jsonl")
|
|
if [row.get("frame_index") for row in semantic_rows] != reference_indices.tolist():
|
|
raise SessionIntegrityError("E23 semantic frame schedule changed")
|
|
semantic_stabilizer = StreamingSemanticStabilizer(profile)
|
|
stabilized_masks = np.stack(
|
|
[semantic_stabilizer.update(mask) for mask in reference_masks]
|
|
).astype(np.uint8, copy=False)
|
|
for row, mask in zip(semantic_rows, stabilized_masks, strict=True):
|
|
if row.get("mask_sha256") != hashlib.sha256(mask.tobytes()).hexdigest():
|
|
raise SessionIntegrityError("E23 semantic mask does not match inline reconstruction")
|
|
row["schema_version"] = "missioncore.e10-semantic-frame/v1"
|
|
row["session_seconds"] = float(frame_times[int(row["frame_index"])]) / 1_000_000_000
|
|
row["temporal_status"] = "e23-inline-spatially-supported-hysteresis"
|
|
|
|
raw_worker_rows = _read_jsonl(worker_root / "raw-fusion-frames.jsonl")
|
|
stable_worker_rows = _read_jsonl(worker_root / "fusion-frames.jsonl")
|
|
fusion_source = {int(row["source_frame_index"]): row for row in stable_worker_rows}
|
|
world_source = {
|
|
int(row["source_frame_index"]): row
|
|
for row in _read_jsonl(worker_root / "world-state.jsonl")
|
|
}
|
|
if set(fusion_source) != set(world_source):
|
|
raise SessionIntegrityError("E23 fusion and world timelines differ")
|
|
fusion_rows: list[dict[str, Any]] = []
|
|
world_rows: list[dict[str, Any]] = []
|
|
dropped_indices: list[int] = []
|
|
for index in range(frame_count):
|
|
session_seconds = float(frame_times[index]) / 1_000_000_000
|
|
fusion = fusion_source.get(index)
|
|
world = world_source.get(index)
|
|
if fusion is None or world is None:
|
|
dropped_indices.append(index)
|
|
fusion_rows.append(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
"fusion_state": "detector-dropped-latest-wins",
|
|
"semantic_source_frame_index": None,
|
|
"semantic_status": "unavailable",
|
|
"objects": [],
|
|
}
|
|
)
|
|
world_rows.append(_dropped_world_row(index, session_seconds))
|
|
continue
|
|
normalized_fusion = json.loads(json.dumps(fusion))
|
|
normalized_fusion.update(
|
|
{
|
|
"schema_version": "missioncore.e10-fusion-frame/v1",
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
normalized_world = json.loads(json.dumps(world))
|
|
normalized_world.update(
|
|
{
|
|
"frame_index": index,
|
|
"source_frame_index": index,
|
|
"session_seconds": session_seconds,
|
|
}
|
|
)
|
|
fusion_rows.append(normalized_fusion)
|
|
world_rows.append(normalized_world)
|
|
expected_drops = int(worker_report["metrics"]["detector"]["queue"]["dropped_overflow"])
|
|
if len(dropped_indices) != expected_drops:
|
|
raise SessionIntegrityError("E23 detector replacement accounting changed")
|
|
|
|
baseline = _quality_metrics(raw_worker_rows, reference_masks)
|
|
stabilized = _quality_metrics(stable_worker_rows, stabilized_masks)
|
|
reductions = {
|
|
"tracking_2d_acceleration_p95_fraction": _fraction_reduction(
|
|
baseline["tracking_2d"]["normalized_acceleration"]["p95"],
|
|
stabilized["tracking_2d"]["normalized_acceleration"]["p95"],
|
|
),
|
|
"tracking_2d_size_step_p95_fraction": _fraction_reduction(
|
|
baseline["tracking_2d"]["normalized_size_step"]["p95"],
|
|
stabilized["tracking_2d"]["normalized_size_step"]["p95"],
|
|
),
|
|
"cuboid_center_step_p95_fraction": _fraction_reduction(
|
|
baseline["cuboids_3d"]["center_step_m"]["p95"],
|
|
stabilized["cuboids_3d"]["center_step_m"]["p95"],
|
|
),
|
|
"cuboid_size_step_p95_fraction": _fraction_reduction(
|
|
baseline["cuboids_3d"]["half_size_step_m"]["p95"],
|
|
stabilized["cuboids_3d"]["half_size_step_m"]["p95"],
|
|
),
|
|
"cuboid_yaw_step_p95_fraction": _fraction_reduction(
|
|
baseline["cuboids_3d"]["yaw_step_degrees"]["p95"],
|
|
stabilized["cuboids_3d"]["yaw_step_degrees"]["p95"],
|
|
),
|
|
"semantic_unsupported_change_fraction": float(
|
|
worker_report["metrics"]["temporal_stability"]["semantic"][
|
|
"unsupported_change_reduction_fraction"
|
|
]
|
|
),
|
|
}
|
|
temporal = worker_report["metrics"]["temporal_stability"]
|
|
acceptance = profile["acceptance"]
|
|
quality_checks = {
|
|
"worker_runtime_accepted": worker_report["state"] == "accepted"
|
|
and all(worker_report["acceptance"]["checks"].values()),
|
|
"source_is_complete_1x": source_report["state"] == "completed"
|
|
and source_report["source"]["speed"] == 1.0,
|
|
"minimum_2d_acceleration_reduction": reductions["tracking_2d_acceleration_p95_fraction"]
|
|
>= float(acceptance["minimum_2d_acceleration_p95_reduction_fraction"]),
|
|
"minimum_3d_center_reduction": reductions["cuboid_center_step_p95_fraction"]
|
|
>= float(acceptance["minimum_3d_center_step_p95_reduction_fraction"]),
|
|
"minimum_3d_yaw_reduction": reductions["cuboid_yaw_step_p95_fraction"]
|
|
>= float(acceptance["minimum_3d_yaw_step_p95_reduction_fraction"]),
|
|
"minimum_semantic_unsupported_change_reduction": reductions[
|
|
"semantic_unsupported_change_fraction"
|
|
]
|
|
>= float(acceptance["minimum_semantic_unsupported_change_reduction_fraction"]),
|
|
"maximum_camera_frame_processing_p95": float(
|
|
worker_report["metrics"]["latency_ms"]["temporal_2d_3d_ms"]["p95"]
|
|
)
|
|
<= float(acceptance["maximum_camera_frame_processing_p95_ms"]),
|
|
"maximum_semantic_frame_processing_p95": float(temporal["semantic"]["processing_ms"]["p95"])
|
|
<= float(acceptance["maximum_semantic_frame_processing_p95_ms"]),
|
|
"maximum_track_states": int(temporal["tracking_2d_3d"]["peak_track_states"])
|
|
<= int(acceptance["maximum_track_states_observed"]),
|
|
"maximum_rss_growth": float(worker_report["metrics"]["runtime_telemetry"]["rss_growth_mib"])
|
|
<= float(acceptance["maximum_rss_growth_mib"]),
|
|
}
|
|
quality = {
|
|
"schema_version": "missioncore.e23-inline-quality/v1",
|
|
"baseline": baseline,
|
|
"stabilized": stabilized,
|
|
"reductions": reductions,
|
|
"checks": quality_checks,
|
|
}
|
|
if not all(quality_checks.values()):
|
|
failed = [name for name, accepted in quality_checks.items() if not accepted]
|
|
raise SessionIntegrityError(f"E23 inline temporal quality failed: {', '.join(failed)}")
|
|
|
|
box_offsets = [0]
|
|
centers: list[list[float]] = []
|
|
half_sizes: list[list[float]] = []
|
|
quaternions: list[list[float]] = []
|
|
colors: list[list[int]] = []
|
|
for row in fusion_rows:
|
|
for item in row["objects"]:
|
|
if not str(item.get("cuboid_status", "")).startswith("accepted-"):
|
|
continue
|
|
centers.append(item["cuboid_center_map"])
|
|
half_sizes.append(item["cuboid_half_size"])
|
|
quaternions.append(item["cuboid_quaternion_xyzw"])
|
|
colors.append(_cuboid_color(item))
|
|
box_offsets.append(len(centers))
|
|
|
|
worker_identity = worker_document["identity"]
|
|
configuration = {
|
|
"pipeline": "e23-inline-temporal-envelope-visual-projection/v1",
|
|
"profile_sha256": profile_sha256,
|
|
"profile": profile,
|
|
"worker_result_id": worker_document["result_id"],
|
|
"source_report": {
|
|
"schema_version": source_report["schema_version"],
|
|
"session_id": source_report["session_id"],
|
|
"speed": source_report["source"]["speed"],
|
|
},
|
|
"semantic_mask_materialization": {
|
|
"mode": "inline-reconstruction-from-immutable-reference-exact-sha256",
|
|
"reference_result_id": reference.result_id,
|
|
"matched_masks": len(semantic_rows),
|
|
},
|
|
"quality": quality,
|
|
}
|
|
selection = {
|
|
"frame_count": frame_count,
|
|
"source_start_frame_index": 0,
|
|
"source_end_frame_index": frame_count - 1,
|
|
"timeline_start_seconds": float(frame_times[0]) / 1_000_000_000,
|
|
"timeline_end_seconds": float(frame_times[-1]) / 1_000_000_000,
|
|
"timeline_sha256": hashlib.sha256(frame_times.tobytes()).hexdigest(),
|
|
}
|
|
identity = {
|
|
"schema_version": "missioncore.e10-integrated-perception-identity/v1",
|
|
"job_id": lab_job.job_id,
|
|
"input_sha256": lab_job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"source_id": lab_job.source_id,
|
|
"lidar_pack_id": lab_pack.name,
|
|
"selection": selection,
|
|
"configuration": configuration,
|
|
"models": worker_identity["models"],
|
|
}
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
result_id = f"e10-integrated-perception-{identity_sha256}"
|
|
destination = results_root / result_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "result.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("E23 LAB visual result id collides")
|
|
return destination, quality
|
|
|
|
staging = _staging_directory(results_root, result_id)
|
|
try:
|
|
semantic_path = staging / "semantic-frames.jsonl"
|
|
fusion_path = staging / "fusion-frames.jsonl"
|
|
world_path = staging / "world-state.jsonl"
|
|
arrays_path = staging / "transient-perception.npz"
|
|
gpu_path = staging / "gpu-telemetry.jsonl"
|
|
report_path = staging / "run-report.json"
|
|
_write_jsonl(semantic_path, semantic_rows)
|
|
_write_jsonl(fusion_path, fusion_rows)
|
|
_write_jsonl(world_path, world_rows)
|
|
np.savez_compressed(
|
|
arrays_path,
|
|
frame_times_ns=frame_times.astype(np.int64, copy=False),
|
|
semantic_frame_indices=reference_indices.astype(np.int64, copy=False),
|
|
semantic_masks=stabilized_masks.astype(np.uint8, copy=False),
|
|
support_offsets=np.zeros(frame_count + 1, dtype=np.int64),
|
|
support_points=np.empty((0, 3), dtype=np.float32),
|
|
support_colors=np.empty((0, 3), dtype=np.uint8),
|
|
box_offsets=np.asarray(box_offsets, dtype=np.int64),
|
|
box_centers=np.asarray(centers, dtype=np.float32).reshape((-1, 3)),
|
|
box_half_sizes=np.asarray(half_sizes, dtype=np.float32).reshape((-1, 3)),
|
|
box_quaternions=np.asarray(quaternions, dtype=np.float32).reshape((-1, 4)),
|
|
box_colors=np.asarray(colors, dtype=np.uint8).reshape((-1, 4)),
|
|
)
|
|
shutil.copyfile(worker_root / "gpu-telemetry.jsonl", gpu_path)
|
|
report = {
|
|
"schema_version": "missioncore.e10-integrated-perception-report/v1",
|
|
"result_id": result_id,
|
|
"created_at_utc": worker_report["created_at_utc"],
|
|
"state": "accepted",
|
|
"ground_truth": False,
|
|
"identity": identity,
|
|
"acceptance": {
|
|
"accepted": True,
|
|
"navigation_or_safety_accepted": False,
|
|
"checks": quality_checks,
|
|
},
|
|
"metrics": {
|
|
**worker_report["metrics"],
|
|
"quality": quality,
|
|
"visual_projection": {
|
|
"frames": frame_count,
|
|
"semantic_masks": len(semantic_rows),
|
|
"detector_replacement_frames": dropped_indices,
|
|
"accepted_cuboids": len(centers),
|
|
},
|
|
},
|
|
"runtime": worker_report.get("runtime", {}),
|
|
"limitations": [
|
|
"This is the accepted E23 recorded 1x inline worker gate, not a physical K1 run.",
|
|
"Latest-wins detector replacements are explicit empty visual frames.",
|
|
"Semantic pixels are reconstructed only after exact inline SHA-256 matches.",
|
|
"LiDAR support points remain in the immutable source scene and are not duplicated.",
|
|
"Navigation and safety authority remain disabled.",
|
|
],
|
|
}
|
|
write_json_atomic(report_path, report)
|
|
artifacts = [
|
|
_artifact_descriptor(
|
|
"e10-semantic-frames",
|
|
semantic_path,
|
|
"missioncore.e10-semantic-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-fusion-frames",
|
|
fusion_path,
|
|
"missioncore.e10-fusion-frame/v1",
|
|
),
|
|
_artifact_descriptor(
|
|
"e10-world-state",
|
|
world_path,
|
|
"missioncore.live-perception-world-state/v1",
|
|
),
|
|
_artifact_descriptor("e10-transient-perception", arrays_path, None),
|
|
_artifact_descriptor("worker-gpu-telemetry", gpu_path, None),
|
|
_artifact_descriptor(
|
|
"e10-run-report",
|
|
report_path,
|
|
"missioncore.e10-integrated-perception-report/v1",
|
|
),
|
|
]
|
|
write_json_atomic(
|
|
staging / "result.json",
|
|
{
|
|
"schema_version": "missioncore.e10-integrated-perception-result/v1",
|
|
"result_id": result_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"created_at_utc": worker_report["created_at_utc"],
|
|
"ground_truth": False,
|
|
"publication_scope": "recorded-integrated-realtime-qualification-only",
|
|
"acceptance_state": "accepted",
|
|
"frames_processed": frame_count,
|
|
"artifacts": artifacts,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination, quality
|
|
|
|
|
|
def _dropped_world_row(frame_index: int, session_seconds: float) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "missioncore.live-perception-world-state/v1",
|
|
"frame_index": frame_index,
|
|
"source_frame_index": frame_index,
|
|
"session_seconds": session_seconds,
|
|
"fusion_state": "detector-dropped-latest-wins",
|
|
"object_count": 0,
|
|
"objects": [],
|
|
"delivery": {
|
|
"health": "degraded",
|
|
"semantic_status": "unavailable",
|
|
"result_age_ms": None,
|
|
"projection_status": "explicit-detector-replacement",
|
|
},
|
|
"coordinate_frames": {
|
|
"sensor_relative": "k1-lidar",
|
|
"vehicle_body": "unavailable-no-rig-to-vehicle-transform",
|
|
"world": "k1-map",
|
|
},
|
|
"clearance": {
|
|
"schema": "diagnostic-polar-obstacle-clearance/v1",
|
|
"state": "unavailable",
|
|
"frame": "k1-lidar",
|
|
"front_m": None,
|
|
"observed_sector_fraction": 0.0,
|
|
"sector_ranges_m": [None] * 72,
|
|
},
|
|
}
|
|
|
|
|
|
def _cuboid_color(item: dict[str, Any]) -> list[int]:
|
|
key = f"e10:{item.get('association_group', 'object')}:{item.get('track_id', 0)}"
|
|
digest = hashlib.sha256(key.encode()).digest()
|
|
return [64 + digest[0] % 176, 64 + digest[1] % 176, 64 + digest[2] % 176, 88]
|
|
|
|
|
|
def _fraction_reduction(baseline: int | float, stabilized: int | float) -> float:
|
|
baseline_value = float(baseline)
|
|
if baseline_value <= 0:
|
|
return 0.0
|
|
return (baseline_value - float(stabilized)) / baseline_value
|
|
|
|
|
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
with path.open(encoding="utf-8") as stream:
|
|
for line in stream:
|
|
value = json.loads(line)
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("LAB JSONL row is not an object")
|
|
rows.append(value)
|
|
return rows
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
|
|
with path.open("x", encoding="utf-8") as stream:
|
|
for row in rows:
|
|
stream.write(
|
|
json.dumps(
|
|
row,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
)
|
|
)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.chmod(path, 0o600)
|
|
|
|
|
|
def _artifact_descriptor(
|
|
kind: str,
|
|
path: Path,
|
|
schema_version: str | None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"kind": kind,
|
|
"path": path.name,
|
|
"schema_version": schema_version,
|
|
"byte_length": path.stat().st_size,
|
|
"sha256": _sha256(path),
|
|
}
|
|
|
|
|
|
def _publish_lab_job(
|
|
source: CameraComputeJob,
|
|
jobs_root: Path,
|
|
lab_session_id: str,
|
|
) -> CameraComputeJob:
|
|
manifest = _read_object(source.manifest_path, source.job_root)
|
|
input_document = json.loads(json.dumps(manifest["input"]))
|
|
input_document["session_id"] = lab_session_id
|
|
input_sha256 = hashlib.sha256(_canonical_json(input_document)).hexdigest()
|
|
job_id = f"recorded-camera-{input_sha256[:24]}"
|
|
destination = jobs_root / job_id
|
|
if destination.exists():
|
|
existing = validate_camera_compute_job(destination)
|
|
if existing.session_id != lab_session_id:
|
|
raise SessionIntegrityError("LAB compute job id collides")
|
|
return existing
|
|
staging = _staging_directory(jobs_root, job_id)
|
|
try:
|
|
_hardlink_tree(source.job_root / "input", staging / "input")
|
|
write_json_atomic(
|
|
staging / "job.json",
|
|
{
|
|
**manifest,
|
|
"job_id": job_id,
|
|
"input_sha256": input_sha256,
|
|
"input": input_document,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return validate_camera_compute_job(destination)
|
|
|
|
|
|
def _publish_lab_pack(
|
|
source: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
packs_root: Path,
|
|
lab_session_id: str,
|
|
) -> Path:
|
|
source_manifest = _read_object(source.pack_root / "manifest.json", source.pack_root)
|
|
identity = json.loads(json.dumps(source_manifest["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
pack_id = f"e10-lidar-pack-{identity_sha256}"
|
|
destination = packs_root / pack_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "manifest.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("LAB LiDAR pack id collides")
|
|
return destination
|
|
staging = _staging_directory(packs_root, pack_id)
|
|
try:
|
|
os.link(
|
|
source.pack_root / "lidar-pack.npz",
|
|
staging / "lidar-pack.npz",
|
|
follow_symlinks=False,
|
|
)
|
|
write_json_atomic(
|
|
staging / "manifest.json",
|
|
{
|
|
**source_manifest,
|
|
"pack_id": pack_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _publish_lab_result(
|
|
source: IntegratedPerceptionResult,
|
|
job: CameraComputeJob,
|
|
pack_root: Path,
|
|
results_root: Path,
|
|
lab_session_id: str,
|
|
) -> Path:
|
|
source_document = _read_object(source.result_root / "result.json", source.result_root)
|
|
identity = json.loads(json.dumps(source_document["identity"]))
|
|
identity.update(
|
|
{
|
|
"job_id": job.job_id,
|
|
"input_sha256": job.input_sha256,
|
|
"session_id": lab_session_id,
|
|
"lidar_pack_id": pack_root.name,
|
|
}
|
|
)
|
|
identity_sha256 = hashlib.sha256(_canonical_json(identity)).hexdigest()
|
|
result_id = f"e10-integrated-perception-{identity_sha256}"
|
|
destination = results_root / result_id
|
|
if destination.exists():
|
|
existing = _read_object(destination / "result.json", destination)
|
|
if existing.get("identity") != identity:
|
|
raise SessionIntegrityError("LAB integrated result id collides")
|
|
return destination
|
|
staging = _staging_directory(results_root, result_id)
|
|
try:
|
|
descriptors: list[dict[str, Any]] = []
|
|
for descriptor in source_document["artifacts"]:
|
|
if descriptor["kind"] == "e10-run-report":
|
|
continue
|
|
source_path = source.result_root / descriptor["path"]
|
|
target = staging / descriptor["path"]
|
|
os.link(source_path, target, follow_symlinks=False)
|
|
descriptors.append(dict(descriptor))
|
|
|
|
report = _read_object(source.report_path, source.result_root)
|
|
report.update(
|
|
{
|
|
"result_id": result_id,
|
|
"identity": identity,
|
|
"lab_projection": {
|
|
"schema_version": "missioncore.integrated-lab-projection/v1",
|
|
"source_session_id": source.job.session_id,
|
|
"source_result_id": source.result_id,
|
|
"lab_session_id": lab_session_id,
|
|
"payloads_recomputed": False,
|
|
},
|
|
}
|
|
)
|
|
report_path = staging / "run-report.json"
|
|
write_json_atomic(report_path, report)
|
|
descriptors.append(
|
|
{
|
|
"kind": "e10-run-report",
|
|
"path": report_path.name,
|
|
"schema_version": "missioncore.e10-integrated-perception-report/v1",
|
|
"byte_length": report_path.stat().st_size,
|
|
"sha256": _sha256(report_path),
|
|
}
|
|
)
|
|
descriptor_order = {
|
|
value["kind"]: index for index, value in enumerate(source_document["artifacts"])
|
|
}
|
|
descriptors.sort(key=lambda value: descriptor_order[value["kind"]])
|
|
write_json_atomic(
|
|
staging / "result.json",
|
|
{
|
|
**source_document,
|
|
"result_id": result_id,
|
|
"identity_sha256": identity_sha256,
|
|
"identity": identity,
|
|
"artifacts": descriptors,
|
|
},
|
|
)
|
|
_publish_directory(staging, destination)
|
|
finally:
|
|
_remove_staging(staging)
|
|
return destination
|
|
|
|
|
|
def _hardlink_tree(source: Path, destination: Path) -> None:
|
|
source_root = source.resolve(strict=True)
|
|
destination.mkdir(mode=0o700, parents=True)
|
|
for current, directories, files in os.walk(source_root):
|
|
current_path = Path(current)
|
|
relative = current_path.relative_to(source_root)
|
|
target_root = destination / relative
|
|
target_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
directories.sort()
|
|
files.sort()
|
|
for filename in files:
|
|
path = current_path / filename
|
|
metadata = path.lstat()
|
|
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
|
|
raise SessionIntegrityError("LAB source tree contains a non-regular file")
|
|
os.link(path, target_root / filename, follow_symlinks=False)
|
|
|
|
|
|
def _staging_directory(root: Path, identity: str) -> Path:
|
|
root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
staging = root / f".{identity}.{secrets.token_hex(12)}.tmp"
|
|
staging.mkdir(mode=0o700)
|
|
return staging
|
|
|
|
|
|
def _publish_directory(staging: Path, destination: Path) -> None:
|
|
try:
|
|
os.replace(staging, destination)
|
|
except OSError:
|
|
if not destination.is_dir():
|
|
raise
|
|
|
|
|
|
def _remove_staging(path: Path) -> None:
|
|
if path.exists():
|
|
shutil.rmtree(path)
|
|
|
|
|
|
def _read_object(path: Path, root: Path) -> dict[str, Any]:
|
|
resolved = path.resolve(strict=True)
|
|
if resolved.parent != root.resolve(strict=True) or resolved.is_symlink():
|
|
raise SessionIntegrityError("LAB manifest escapes its immutable root")
|
|
value = json.loads(resolved.read_text(encoding="utf-8"))
|
|
if not isinstance(value, dict):
|
|
raise SessionIntegrityError("LAB manifest is not a JSON object")
|
|
return value
|
|
|
|
|
|
def _canonical_json(value: object) -> bytes:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|