feat(lab): stabilize autonomous TGS playback

This commit is contained in:
DCCONSTRUCTIONS
2026-08-27 20:34:20 +03:00
parent c8593207d3
commit 95a1ef5057
26 changed files with 3326 additions and 253 deletions
@@ -0,0 +1,538 @@
"""Build and verify autonomous chunked M49 physical-safety playback artifacts."""
from __future__ import annotations
import hashlib
import json
import math
import shutil
import struct
import tempfile
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Final
import numpy as np
SCHEMA: Final = "missioncore.m49-physical-safety-playback/v1"
PROFILE_SCHEMA: Final = "missioncore.m49-physical-safety-shadow-profile/v1"
PREFIX: Final = "m49-physical-safety-playback-"
CHUNK_MAGIC: Final = b"MCPSCH01"
CHUNK_HEADER: Final = struct.Struct("<8sIIIII")
SOURCE_TRACKS: Final = (
"costmap-cell-centers-xy-m.npy",
"costmap-states.npy",
"costmap-z-bounds-m.npy",
"frames.ndjson",
)
_HASH_BLOCK_BYTES: Final = 1024 * 1024
_MAX_JSON_BYTES: Final = 4 * 1024 * 1024
class M49PhysicalSafetyPlaybackError(RuntimeError):
"""The autonomous playback artifact is missing or violates its contract."""
@dataclass(frozen=True, slots=True)
class M49PhysicalSafetyPlayback:
result_id: str
root: Path
manifest: dict[str, Any]
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for block in iter(lambda: stream.read(_HASH_BLOCK_BYTES), b""):
digest.update(block)
return digest.hexdigest()
def _canonical_sha256(value: object) -> str:
content = json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
return hashlib.sha256(content).hexdigest()
def _json(path: Path, label: str) -> dict[str, Any]:
if path.is_symlink() or not path.is_file() or path.stat().st_size > _MAX_JSON_BYTES:
raise M49PhysicalSafetyPlaybackError(f"{label} is unavailable")
try:
value = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
raise M49PhysicalSafetyPlaybackError(f"{label} is invalid") from error
if not isinstance(value, dict):
raise M49PhysicalSafetyPlaybackError(f"{label} is invalid")
return value
def _artifact(path: Path, *, media_type: str, role: str) -> dict[str, object]:
return {
"path": path.name,
"media_type": media_type,
"role": role,
"byte_length": path.stat().st_size,
"sha256": _sha256(path),
}
def _source_artifacts(source: Path, manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, list):
raise M49PhysicalSafetyPlaybackError("source artifact catalog is unavailable")
result: dict[str, dict[str, Any]] = {}
for value in artifacts:
if not isinstance(value, dict) or not isinstance(value.get("path"), str):
raise M49PhysicalSafetyPlaybackError("source artifact catalog is invalid")
name = value["path"]
if name in (*SOURCE_TRACKS, "report.json"):
path = source / name
if (
path.parent != source
or path.is_symlink()
or not path.is_file()
or value.get("byte_length") != path.stat().st_size
or value.get("sha256") != _sha256(path)
):
raise M49PhysicalSafetyPlaybackError(f"source artifact changed: {name}")
result[name] = value
if set(result) != {*SOURCE_TRACKS, "report.json"}:
raise M49PhysicalSafetyPlaybackError("source spatial tracks are incomplete")
return result
def _state_codes(report: dict[str, Any]) -> dict[str, int]:
candidates = (
report.get("state_codes"),
report.get("visual_review", {}).get("state_codes")
if isinstance(report.get("visual_review"), dict)
else None,
)
for candidate in candidates:
if not isinstance(candidate, dict):
continue
parsed = {
str(name): int(code)
for name, code in candidate.items()
if isinstance(name, str)
and isinstance(code, int)
and not isinstance(code, bool)
and 0 <= code <= 255
}
if len(parsed) == len(candidate) and parsed.get("UNOBSERVED") == 0:
return parsed
raise M49PhysicalSafetyPlaybackError("source state codes are unavailable")
def _frame_rows(path: Path) -> tuple[dict[str, Any], ...]:
rows: list[dict[str, Any]] = []
try:
with path.open("r", encoding="utf-8") as stream:
for sequence, line in enumerate(stream):
value = json.loads(line)
if (
not isinstance(value, dict)
or not isinstance(value.get("sample_available"), bool)
or not isinstance(value.get("source_frame_index"), int)
or isinstance(value.get("source_frame_index"), bool)
or not isinstance(value.get("session_seconds"), (int, float))
or isinstance(value.get("session_seconds"), bool)
or not math.isfinite(float(value["session_seconds"]))
):
raise M49PhysicalSafetyPlaybackError(
f"source frame catalog changed at sequence {sequence}"
)
rows.append(value)
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
raise M49PhysicalSafetyPlaybackError("source frame catalog is invalid") from error
if not rows:
raise M49PhysicalSafetyPlaybackError("source frame catalog is empty")
return tuple(rows)
def _source_geometry(report: dict[str, Any]) -> tuple[float, float, str]:
configuration = report.get("configuration")
if not isinstance(configuration, dict):
raise M49PhysicalSafetyPlaybackError("source geometry is unavailable")
cell_size = configuration.get("cell_size_m")
radius = configuration.get("radius_m")
coordinate_frame = configuration.get("coordinate_frame")
if (
not isinstance(cell_size, (int, float))
or isinstance(cell_size, bool)
or not math.isfinite(float(cell_size))
or float(cell_size) <= 0
or not isinstance(radius, (int, float))
or isinstance(radius, bool)
or not math.isfinite(float(radius))
or float(radius) <= 0
or coordinate_frame != "map-gravity-local"
):
raise M49PhysicalSafetyPlaybackError("source geometry changed")
return float(cell_size), float(radius), coordinate_frame
def _profile_policy(profile: dict[str, Any], cell_size_m: float) -> dict[str, int]:
visual = profile.get("operator_visual")
terrain = profile.get("terrain_evidence")
if (
profile.get("schema_version") != PROFILE_SCHEMA
or not isinstance(visual, dict)
or not isinstance(terrain, dict)
or visual.get("worker_role") != "realtime-only"
or visual.get("worker_runtime_dependency") is not False
or visual.get("sealed_artifact_required") is not True
):
raise M49PhysicalSafetyPlaybackError("physical-safety profile changed")
playback = visual.get("playback_buffer")
admitted_sizes = [
terrain.get("accepted_baseline_cell_size_m"),
*(terrain.get("challenger_cell_sizes_m") or []),
]
if (
not isinstance(playback, dict)
or playback.get("chunk_frame_count") != 32
or playback.get("startup_prebuffer_chunk_count") != 2
or playback.get("resident_chunk_count_max") != 3
or not any(
isinstance(value, (int, float))
and not isinstance(value, bool)
and math.isclose(float(value), cell_size_m, abs_tol=1e-9)
for value in admitted_sizes
)
):
raise M49PhysicalSafetyPlaybackError("physical-safety playback policy changed")
return {
"chunk_frame_count": 32,
"startup_prebuffer_chunk_count": 2,
"resident_chunk_count_max": 3,
"forward_prefetch_chunk_count": 1,
}
def _chunk_payload(
path: Path,
*,
start: int,
states: np.ndarray,
z_bounds: np.ndarray,
) -> None:
frame_count, cell_count = states.shape
state_values = np.ascontiguousarray(states, dtype=np.uint8)
z_values = np.ascontiguousarray(z_bounds, dtype="<f4")
state_bytes = state_values.nbytes
z_bytes = z_values.nbytes
padding = (4 - ((CHUNK_HEADER.size + state_bytes) % 4)) % 4
with path.open("wb") as stream:
stream.write(
CHUNK_HEADER.pack(
CHUNK_MAGIC,
start,
frame_count,
cell_count,
state_bytes,
z_bytes,
)
)
stream.write(memoryview(state_values).cast("B"))
if padding:
stream.write(b"\0" * padding)
stream.write(memoryview(z_values).cast("B"))
def seal_m49_physical_safety_playback(
*,
source_root: Path,
destination_root: Path,
profile_path: Path,
created_at_utc: str | None = None,
) -> M49PhysicalSafetyPlayback:
"""Sequentially seal local tracks without starting or contacting Worker 006."""
raw_source = source_root.expanduser().absolute()
if raw_source.is_symlink() or not raw_source.is_dir():
raise M49PhysicalSafetyPlaybackError("sealed source root is unavailable")
source = raw_source.resolve(strict=True)
destination = destination_root.expanduser().absolute()
destination.mkdir(parents=True, exist_ok=True)
if destination.is_symlink():
raise M49PhysicalSafetyPlaybackError("destination must not be a symlink")
raw_profile = profile_path.expanduser().absolute()
if raw_profile.is_symlink() or not raw_profile.is_file():
raise M49PhysicalSafetyPlaybackError("physical-safety profile is unavailable")
profile_file = raw_profile.resolve(strict=True)
source_manifest = _json(source / "manifest.json", "source manifest")
source_report = _json(source / "report.json", "source report")
source_result_id = source_manifest.get("result_id")
source_identity = source_manifest.get("identity")
source_identity_sha256 = source_manifest.get("identity_sha256")
if (
not isinstance(source_result_id, str)
or source_report.get("result_id") != source_result_id
or not isinstance(source_identity, dict)
or not isinstance(source_identity_sha256, str)
or source_identity_sha256 != _canonical_sha256(source_identity)
or not source_result_id.endswith(source_identity_sha256)
):
raise M49PhysicalSafetyPlaybackError("source result identity changed")
source_artifacts = _source_artifacts(source, source_manifest)
state_codes = _state_codes(source_report)
cell_size_m, radius_m, coordinate_frame = _source_geometry(source_report)
profile = _json(profile_file, "physical-safety profile")
policy = _profile_policy(profile, cell_size_m)
profile_sha256 = _sha256(profile_file)
source_manifest_sha256 = _sha256(source / "manifest.json")
try:
centers = np.load(
source / "costmap-cell-centers-xy-m.npy", mmap_mode="r", allow_pickle=False
)
states = np.load(source / "costmap-states.npy", mmap_mode="r", allow_pickle=False)
z_bounds = np.load(
source / "costmap-z-bounds-m.npy", mmap_mode="r", allow_pickle=False
)
except (OSError, ValueError) as error:
raise M49PhysicalSafetyPlaybackError("source numeric tracks are invalid") from error
frames = _frame_rows(source / "frames.ndjson")
if (
centers.dtype != np.dtype("<f4")
or states.dtype != np.dtype("uint8")
or z_bounds.dtype != np.dtype("<f4")
or centers.ndim != 2
or centers.shape[1] != 2
or states.ndim != 2
or z_bounds.ndim != 3
or z_bounds.shape[2] != 2
or states.shape[0] != len(frames)
or states.shape[1] != centers.shape[0]
or z_bounds.shape != (states.shape[0], states.shape[1], 2)
or not np.isfinite(centers).all()
):
raise M49PhysicalSafetyPlaybackError("source numeric shape or dtype changed")
frame_count, cell_count = states.shape
admitted_codes = np.asarray(sorted(set(state_codes.values())), dtype=np.uint8)
identity = {
"schema_version": SCHEMA,
"source_result_id": source_result_id,
"source_manifest_sha256": source_manifest_sha256,
"source_track_sha256": {
name: source_artifacts[name]["sha256"] for name in SOURCE_TRACKS
},
"profile_id": profile.get("profile_id"),
"profile_sha256": profile_sha256,
"coordinate_frame": coordinate_frame,
"frame_count": frame_count,
"cell_count": cell_count,
"cell_size_m": cell_size_m,
"radius_m": radius_m,
"state_codes": state_codes,
"playback_policy": policy,
"authority": {
"commands_enabled": False,
"navigation_or_safety_accepted": False,
"actuation_accepted": False,
},
}
identity_sha256 = _canonical_sha256(identity)
result_id = f"{PREFIX}{identity_sha256}"
target = destination / result_id
if target.exists():
existing = read_m49_physical_safety_playback(target)
existing_playback = existing.manifest["playback"]
assert isinstance(existing_playback, dict)
existing_chunks = existing_playback["chunks"]
assert isinstance(existing_chunks, list)
for value in (
existing_playback["centers"],
existing_playback["frames"],
*existing_chunks,
):
assert isinstance(value, dict)
verify_m49_physical_safety_artifact(existing, value)
return existing
created = created_at_utc or datetime.now(tz=UTC).isoformat().replace("+00:00", "Z")
chunk_frame_count = int(policy["chunk_frame_count"])
with tempfile.TemporaryDirectory(
prefix="mission-core-m49-physical-safety-", dir=destination
) as raw_staging:
staging = Path(raw_staging) / result_id
staging.mkdir()
centers_path = staging / "centers.f32"
np.ascontiguousarray(centers, dtype="<f4").tofile(centers_path)
frames_path = staging / "frames.ndjson"
shutil.copyfile(source / "frames.ndjson", frames_path)
centers_artifact = _artifact(
centers_path,
media_type="application/octet-stream",
role="costmap-cell-centers-f32le",
)
frames_artifact = _artifact(
frames_path,
media_type="application/x-ndjson",
role="frame-catalog",
)
chunks: list[dict[str, object]] = []
for index, start in enumerate(range(0, frame_count, chunk_frame_count)):
count = min(chunk_frame_count, frame_count - start)
state_slice = states[start : start + count]
if not np.isin(state_slice, admitted_codes).all():
raise M49PhysicalSafetyPlaybackError(
f"source state code changed in chunk {index}"
)
for local_index, frame in enumerate(frames[start : start + count]):
if frame["sample_available"] is False and (
np.any(state_slice[local_index] != 0)
or np.isfinite(z_bounds[start + local_index]).any()
):
raise M49PhysicalSafetyPlaybackError(
"missing source sample did not remain fail-closed"
)
chunk_path = staging / f"chunk-{index:06d}.bin"
_chunk_payload(
chunk_path,
start=start,
states=state_slice,
z_bounds=z_bounds[start : start + count],
)
descriptor = _artifact(
chunk_path,
media_type="application/octet-stream",
role="states-u8-and-z-bounds-f32le",
)
descriptor.update(
{
"index": index,
"start": start,
"count": count,
"header_bytes": CHUNK_HEADER.size,
"format": "mcpsch01-states-u8-aligned-z-bounds-f32le",
}
)
chunks.append(descriptor)
manifest = {
"schema_version": SCHEMA,
"result_id": result_id,
"created_at_utc": created,
"identity_sha256": identity_sha256,
"identity": identity,
"execution": {
"execution_class": "local-sequential-offline",
"worker_role": "realtime-only",
"worker_runtime_dependency": False,
"worker_requests_required": 0,
},
"playback": {
"coordinate_frame": coordinate_frame,
"source_pace_hz": profile["operator_visual"]["source_pace_hz"],
"frame_count": frame_count,
"cell_count": cell_count,
"cell_size_m": cell_size_m,
"radius_m": radius_m,
"state_codes": state_codes,
**policy,
"centers": {
**centers_artifact,
"dtype": "<f4",
"shape": [cell_count, 2],
},
"frames": {
**frames_artifact,
"dtype": "ndjson",
"shape": [frame_count],
},
"chunks": chunks,
},
"authority": identity["authority"],
}
(staging / "manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
encoding="utf-8",
)
staging.replace(target)
return read_m49_physical_safety_playback(target)
def read_m49_physical_safety_playback(root: Path) -> M49PhysicalSafetyPlayback:
raw_candidate = root.expanduser().absolute()
if raw_candidate.is_symlink() or not raw_candidate.is_dir():
raise M49PhysicalSafetyPlaybackError("physical-safety playback root is invalid")
candidate = raw_candidate.resolve(strict=True)
manifest = _json(candidate / "manifest.json", "physical-safety playback manifest")
identity = manifest.get("identity")
execution = manifest.get("execution")
if (
manifest.get("schema_version") != SCHEMA
or not isinstance(identity, dict)
or not isinstance(execution, dict)
or manifest.get("identity_sha256") != _canonical_sha256(identity)
or manifest.get("result_id") != candidate.name
or candidate.name != f"{PREFIX}{manifest.get('identity_sha256')}"
or execution.get("worker_requests_required") != 0
or execution.get("worker_runtime_dependency") is not False
):
raise M49PhysicalSafetyPlaybackError("physical-safety playback identity changed")
playback = manifest.get("playback")
if not isinstance(playback, dict) or not isinstance(playback.get("chunks"), list):
raise M49PhysicalSafetyPlaybackError("physical-safety playback catalog changed")
descriptors = [playback.get("centers"), playback.get("frames"), *playback["chunks"]]
seen: set[str] = set()
for descriptor in descriptors:
if not isinstance(descriptor, dict) or not isinstance(descriptor.get("path"), str):
raise M49PhysicalSafetyPlaybackError("physical-safety artifact entry changed")
name = descriptor["path"]
path = candidate / name
if (
name in seen
or path.parent != candidate
or path.is_symlink()
or not path.is_file()
or descriptor.get("byte_length") != path.stat().st_size
or not isinstance(descriptor.get("sha256"), str)
or len(descriptor["sha256"]) != 64
or any(character not in "0123456789abcdef" for character in descriptor["sha256"])
):
raise M49PhysicalSafetyPlaybackError("physical-safety artifact changed")
seen.add(name)
return M49PhysicalSafetyPlayback(candidate.name, candidate, manifest)
def verify_m49_physical_safety_artifact(
result: M49PhysicalSafetyPlayback,
descriptor: dict[str, Any],
) -> Path:
name = descriptor.get("path")
if not isinstance(name, str):
raise M49PhysicalSafetyPlaybackError("physical-safety artifact path changed")
path = result.root / name
if (
path.parent != result.root
or path.is_symlink()
or not path.is_file()
or descriptor.get("byte_length") != path.stat().st_size
or descriptor.get("sha256") != _sha256(path)
):
raise M49PhysicalSafetyPlaybackError("physical-safety artifact digest changed")
return path
__all__ = [
"CHUNK_HEADER",
"CHUNK_MAGIC",
"M49PhysicalSafetyPlayback",
"M49PhysicalSafetyPlaybackError",
"PREFIX",
"SCHEMA",
"read_m49_physical_safety_playback",
"seal_m49_physical_safety_playback",
"verify_m49_physical_safety_artifact",
]