feat(perception): qualify 1x realtime replay envelope

experiments/perception/worker/Invoke-E15PersistentShadowRun.ps1

@@ -6,6 +6,7 @@ param(
     [string]$PersistentContainer = "mission-core-perception-worker",
     [string]$PersistentOutputRoot = "D:\NDC_MISSIONCORE\runtime\derived\.perception-persistent-publish",
     [ValidateRange(1024, 65535)] [int]$PersistentPort = 18020,
+    [ValidateRange(5, 3600)] [int]$MaximumDurationSeconds = 20,
     [ValidateRange(1, 1000)] [int]$FreeGiBFloor = 360
 )
 
@@ -48,6 +49,7 @@ $request = @{
     request_id = $RequestId
     output_name = $outputName
     token = $token
+    max_duration_seconds = $MaximumDurationSeconds
 } | ConvertTo-Json -Compress
 $token = $null
 $client = (

experiments/perception/worker/e15_shadow_runtime.py

@@ -184,6 +184,7 @@ class PersistentFmp4Decoder:
         self._started = False
         self._init_seen = False
         self._last_source_sequence: int | None = None
+        self._source_sequence_gaps = 0
         self._decoded_frames = 0
         self._failure: BaseException | None = None
 
@@ -202,11 +203,18 @@ class PersistentFmp4Decoder:
     def feed_segment(self, metadata: CameraFragmentMetadata, payload: bytes) -> None:
         if not self._init_seen:
             raise ShadowRuntimeError("camera media arrived before init")
-        expected = 1 if self._last_source_sequence is None else self._last_source_sequence + 1
-        if metadata.source_sequence != expected:
+        if metadata.source_sequence < 1:
+            raise ShadowRuntimeError("camera source sequence must be positive")
+        if (
+            self._last_source_sequence is not None
+            and metadata.source_sequence <= self._last_source_sequence
+        ):
             raise ShadowRuntimeError(
-                f"camera source sequence gap: expected {expected}, got {metadata.source_sequence}"
+                "camera source sequence is not increasing: "
+                f"previous {self._last_source_sequence}, got {metadata.source_sequence}"
             )
+        expected = 1 if self._last_source_sequence is None else self._last_source_sequence + 1
+        self._source_sequence_gaps += metadata.source_sequence - expected
         self._metadata.publish(metadata)
         try:
             self._media.append(payload)
@@ -234,6 +242,7 @@ class PersistentFmp4Decoder:
         return {
             "init_seen": self._init_seen,
             "last_source_sequence": self._last_source_sequence,
+            "source_sequence_gaps": self._source_sequence_gaps,
             "decoded_frames": self._decoded_frames,
             "failed": self._failure is not None,
             "media": self._media.snapshot(),

experiments/perception/worker/run_e15_shadow_inference.py

@@ -8,6 +8,7 @@ import hashlib
 import importlib.metadata
 import io
 import json
+import math
 import os
 import platform
 import queue
@@ -347,6 +348,7 @@ class _TransportState:
     last_ingress_sequence: int | None = None
     ingress_sequence_gaps: int = 0
     camera_sequence_gaps: int = 0
+    last_camera_source_sequence: int | None = None
     session_id: str | None = None
     session_end_seen: bool = False
     timed_out: bool = False
@@ -360,6 +362,175 @@ class _TransportState:
             self.failures = []
 
 
+class _RuntimeTelemetry:
+    """Sample process, host-memory and bounded-queue state during one warm run."""
+
+    def __init__(
+        self,
+        stream: Any,
+        *,
+        interval_seconds: float,
+        snapshotters: dict[str, Any],
+    ) -> None:
+        if not 0.25 <= interval_seconds <= 10:
+            raise RuntimeError("runtime telemetry interval is outside bounds")
+        self._stream = stream
+        self._interval = interval_seconds
+        self._snapshotters = snapshotters
+        self._stop = threading.Event()
+        self._thread = threading.Thread(
+            target=self._run,
+            name="runtime-telemetry",
+            daemon=True,
+        )
+        self._started = time.perf_counter()
+        self._previous_wall = self._started
+        self._previous_cpu = time.process_time()
+        self.samples: list[dict[str, Any]] = []
+
+    def __enter__(self) -> _RuntimeTelemetry:
+        self._thread.start()
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self._stop.set()
+        self._thread.join(timeout=self._interval + 5)
+        self._sample()
+
+    @staticmethod
+    def _read_number(path: str) -> int | None:
+        try:
+            value = Path(path).read_text(encoding="ascii").strip()
+            if value == "max":
+                return None
+            return int(value)
+        except (OSError, ValueError):
+            return None
+
+    @staticmethod
+    def _process_status() -> tuple[float, int]:
+        rss_mib = 0.0
+        thread_count = 0
+        try:
+            fields = Path("/proc/self/statm").read_text(encoding="ascii").split()
+            rss_mib = int(fields[1]) * os.sysconf("SC_PAGE_SIZE") / 2**20
+        except (OSError, ValueError, IndexError):
+            pass
+        try:
+            for line in Path("/proc/self/status").read_text(encoding="ascii").splitlines():
+                if line.startswith("Threads:"):
+                    thread_count = int(line.split(":", 1)[1].strip())
+                    break
+        except (OSError, ValueError):
+            pass
+        return rss_mib, thread_count
+
+    @staticmethod
+    def _memory_status() -> tuple[float, float]:
+        total_kib = 0
+        available_kib = 0
+        try:
+            for line in Path("/proc/meminfo").read_text(encoding="ascii").splitlines():
+                if line.startswith("MemTotal:"):
+                    total_kib = int(line.split()[1])
+                elif line.startswith("MemAvailable:"):
+                    available_kib = int(line.split()[1])
+        except (OSError, ValueError, IndexError):
+            pass
+        return total_kib / 1024, available_kib / 1024
+
+    @staticmethod
+    def _io_status() -> tuple[int, int]:
+        read_bytes = 0
+        write_bytes = 0
+        try:
+            for line in Path("/proc/self/io").read_text(encoding="ascii").splitlines():
+                if line.startswith("read_bytes:"):
+                    read_bytes = int(line.split(":", 1)[1].strip())
+                elif line.startswith("write_bytes:"):
+                    write_bytes = int(line.split(":", 1)[1].strip())
+        except (OSError, ValueError):
+            pass
+        return read_bytes, write_bytes
+
+    def _sample(self) -> None:
+        now = time.perf_counter()
+        process_cpu = time.process_time()
+        wall_delta = max(1e-9, now - self._previous_wall)
+        cpu_percent = max(0.0, (process_cpu - self._previous_cpu) / wall_delta * 100)
+        self._previous_wall = now
+        self._previous_cpu = process_cpu
+        rss_mib, thread_count = self._process_status()
+        memory_total_mib, memory_available_mib = self._memory_status()
+        read_bytes, write_bytes = self._io_status()
+        cgroup_current = self._read_number("/sys/fs/cgroup/memory.current")
+        cgroup_limit = self._read_number("/sys/fs/cgroup/memory.max")
+        queues: dict[str, Any] = {}
+        for name, snapshotter in self._snapshotters.items():
+            try:
+                queues[name] = snapshotter()
+            except (RuntimeError, ValueError, OSError):
+                queues[name] = {"state": "snapshot-unavailable"}
+        sample = {
+            "schema_version": "missioncore.worker-runtime-telemetry/v1",
+            "elapsed_seconds": round(now - self._started, 6),
+            "process_cpu_percent": round(cpu_percent, 6),
+            "process_rss_mib": round(rss_mib, 6),
+            "process_threads": thread_count,
+            "process_read_bytes": read_bytes,
+            "process_write_bytes": write_bytes,
+            "system_memory_total_mib": round(memory_total_mib, 6),
+            "system_memory_available_mib": round(memory_available_mib, 6),
+            "cgroup_memory_current_mib": (
+                round(cgroup_current / 2**20, 6) if cgroup_current is not None else None
+            ),
+            "cgroup_memory_limit_mib": (
+                round(cgroup_limit / 2**20, 6) if cgroup_limit is not None else None
+            ),
+            "queues": queues,
+        }
+        self.samples.append(sample)
+        self._stream.write(json.dumps(sample, sort_keys=True) + "\n")
+        self._stream.flush()
+
+    def _run(self) -> None:
+        while not self._stop.wait(self._interval):
+            self._sample()
+
+    def summary(self) -> dict[str, Any]:
+        numeric_fields = (
+            "process_cpu_percent",
+            "process_rss_mib",
+            "process_threads",
+            "system_memory_available_mib",
+            "cgroup_memory_current_mib",
+        )
+        summary: dict[str, Any] = {
+            "sample_count": len(self.samples),
+            "interval_seconds": self._interval,
+        }
+        for field in numeric_fields:
+            values = [
+                float(sample[field])
+                for sample in self.samples
+                if isinstance(sample.get(field), int | float)
+            ]
+            summary[field] = _percentiles(values)
+        if self.samples:
+            quarter = max(1, len(self.samples) // 4)
+            early = [float(value["process_rss_mib"]) for value in self.samples[:quarter]]
+            late = [float(value["process_rss_mib"]) for value in self.samples[-quarter:]]
+            summary["rss_growth_mib"] = round(
+                float(_percentiles(late)["p95"]) - float(_percentiles(early)["p95"]),
+                6,
+            )
+            summary["final_queues"] = self.samples[-1]["queues"]
+        else:
+            summary["rss_growth_mib"] = 0.0
+            summary["final_queues"] = {}
+        return summary
+
+
 def _send_client_binary_frame(stream: Any, payload: bytes) -> None:
     if not payload or len(payload) > LIVE_RESULT_MAX_PAYLOAD_BYTES + 256 * 1024 + 8:
         raise ShadowRuntimeError("live result websocket frame exceeds the bound")
@@ -478,20 +649,31 @@ def _receiver(
             elif modality == "camera-init":
                 decoder.feed_init(payload)
             elif modality == "camera-frame":
-                try:
-                    decoder.feed_segment(
-                        CameraFragmentMetadata(
-                            ingress_sequence=sequence,
-                            source_sequence=int(header["source_sequence"]),
-                            captured_at_epoch_ns=int(header["captured_at_epoch_ns"]),
-                            worker_received_monotonic=worker_received,
-                        ),
-                        payload,
+                camera_source_sequence = int(header["source_sequence"])
+                expected_camera_sequence = (
+                    1
+                    if state.last_camera_source_sequence is None
+                    else state.last_camera_source_sequence + 1
+                )
+                if camera_source_sequence < expected_camera_sequence:
+                    raise ShadowRuntimeError(
+                        "camera source sequence is not increasing: "
+                        f"expected at least {expected_camera_sequence}, "
+                        f"got {camera_source_sequence}"
                     )
-                except ShadowRuntimeError as exc:
-                    if "source sequence gap" in str(exc):
-                        state.camera_sequence_gaps += 1
-                    raise
+                state.camera_sequence_gaps += (
+                    camera_source_sequence - expected_camera_sequence
+                )
+                state.last_camera_source_sequence = camera_source_sequence
+                decoder.feed_segment(
+                    CameraFragmentMetadata(
+                        ingress_sequence=sequence,
+                        source_sequence=camera_source_sequence,
+                        captured_at_epoch_ns=int(header["captured_at_epoch_ns"]),
+                        worker_received_monotonic=worker_received,
+                    ),
+                    payload,
+                )
             elif modality in {"lidar", "pose"}:
                 decode_started = time.perf_counter()
                 normalized = normalize_k1_message(
@@ -729,10 +911,11 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
     )
     first_camera_epoch_ns: list[int] = []
     last_camera_epoch_ns: list[int] = []
-    decoded_frames: list[DecodedCameraFrame] = []
+    decoded_frame_count = 0
     callback_failures: list[BaseException] = []
 
     def on_decoded(frame: DecodedCameraFrame) -> None:
+        nonlocal decoded_frame_count
         try:
             if not first_camera_epoch_ns:
                 first_camera_epoch_ns.append(frame.metadata.captured_at_epoch_ns)
@@ -756,7 +939,7 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
                 decode_ms=frame.decode_age_ms,
                 source_release_lag_ms=0.0,
             )
-            decoded_frames.append(frame)
+            decoded_frame_count += 1
             detector_queue.publish(envelope)
             if frame.frame_index % int(scheduling["semantic_sample_every_frames"]) == 0:
                 semantic_queue.publish(envelope)
@@ -776,7 +959,6 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
     result_queue: queue.Queue[bytes] = queue.Queue(maxsize=2)
     result_complete = threading.Event()
     sensor_decode_ms = {"lidar": [], "pose": []}
-    completed_semantics: list[Any] = []
     semantic_errors: list[BaseException] = []
     semantic_latency = {
         name: []
@@ -819,15 +1001,59 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
     fusion_path = output / "fusion-frames.jsonl"
     world_path = output / "world-state.jsonl"
     gpu_path = output / "gpu-telemetry.jsonl"
+    runtime_path = output / "runtime-telemetry.jsonl"
     run_started = time.perf_counter()
+    process_cpu_started = time.process_time()
 
     with (
         semantic_path.open("x", encoding="utf-8", newline="\n") as semantic_stream,
         fusion_path.open("x", encoding="utf-8", newline="\n") as fusion_stream,
         world_path.open("x", encoding="utf-8", newline="\n") as world_stream,
         gpu_path.open("x", encoding="utf-8", newline="\n") as gpu_stream,
+        runtime_path.open("x", encoding="utf-8", newline="\n") as runtime_stream,
         _GpuTelemetry(gpu_stream, 1.0) as gpu,
+        _RuntimeTelemetry(
+            runtime_stream,
+            interval_seconds=1.0,
+            snapshotters={
+                "detector": detector_queue.snapshot,
+                "semantic": semantic_queue.snapshot,
+                "decoder": decoder.snapshot,
+                "synchronizer": synchronizer.snapshot,
+                "result": lambda: {
+                    "capacity": result_queue.maxsize,
+                    "depth": result_queue.qsize(),
+                    "dropped_overflow": transport.results_dropped,
+                    "published": transport.results_published,
+                },
+            },
+        ) as runtime_telemetry,
     ):
+        class SemanticResultStream:
+            def __init__(self) -> None:
+                self.count = 0
+
+            def append(self, result: Any) -> None:
+                semantic_stream.write(
+                    json.dumps(
+                        {
+                            "schema_version": SEMANTIC_SCHEMA,
+                            "frame_index": result.frame_index,
+                            "source_frame_index": result.source_frame_index,
+                            "session_seconds": result.session_seconds,
+                            "completion_age_ms": result.completion_age_ms,
+                            "mask_sha256": result.mask_sha256,
+                            "class_pixels": result.class_pixels,
+                        },
+                        sort_keys=True,
+                        separators=(",", ":"),
+                        allow_nan=False,
+                    )
+                    + "\n"
+                )
+                self.count += 1
+
+        completed_semantics = SemanticResultStream()
         semantic_thread = threading.Thread(
             target=semantic_worker,
             kwargs={
@@ -1101,25 +1327,17 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
             or callback_failures
             or semantic_errors
         ):
-            raise RuntimeError("LAB E15 runtime worker failed")
-        for result in completed_semantics:
-            semantic_stream.write(
-                json.dumps(
-                    {
-                        "schema_version": SEMANTIC_SCHEMA,
-                        "frame_index": result.frame_index,
-                        "source_frame_index": result.source_frame_index,
-                        "session_seconds": result.session_seconds,
-                        "completion_age_ms": result.completion_age_ms,
-                        "mask_sha256": result.mask_sha256,
-                        "class_pixels": result.class_pixels,
-                    },
-                    sort_keys=True,
-                    separators=(",", ":"),
-                    allow_nan=False,
-                )
-                + "\n"
+            failures = [
+                *(transport.failures or []),
+                *decoder_watch_failures,
+                *callback_failures,
+                *semantic_errors,
+            ]
+            summary = "; ".join(
+                f"{type(failure).__name__}: {str(failure)[:240]}"
+                for failure in failures[:8]
             )
+            raise RuntimeError(f"LAB E15 runtime worker failed: {summary}")
         for stream in (semantic_stream, fusion_stream, world_stream):
             stream.flush()
             os.fsync(stream.fileno())
@@ -1136,7 +1354,8 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
     detector_fps = int(detector_state["consumed"]) / max(source_span, run_wall, 1e-9)
     semantic_fps = int(semantic_state["consumed"]) / max(source_span, run_wall, 1e-9)
     semantic_scheduled = (
-        (len(decoded_frames) - 1) // int(scheduling["semantic_sample_every_frames"])
+        (decoded_frame_count - 1)
+        // int(scheduling["semantic_sample_every_frames"])
     ) + 1
     fresh_coverage = status_counts["fresh"] / max(1, int(detector_state["consumed"]))
     fused_fraction = fused_frames / max(1, int(detector_state["consumed"]))
@@ -1149,17 +1368,17 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
     }
     acceptance = live["acceptance"]
     checks = {
-        "minimum_camera_frames": len(decoded_frames)
+        "minimum_camera_frames": decoded_frame_count
         >= int(acceptance["minimum_camera_frames"]),
         "camera_decoder_accounting": decoder.snapshot()["decoded_frames"]
         == transport.counts["camera-frame"],
         "detector_accounting": int(detector_state["consumed"])
         + int(detector_state["dropped_overflow"])
-        == len(decoded_frames),
+        == decoded_frame_count,
         "detector_minimum_effective_fps": detector_fps
         >= float(acceptance["detector_minimum_effective_fps"]),
         "detector_maximum_drop_fraction": int(detector_state["dropped_overflow"])
-        / len(decoded_frames)
+        / decoded_frame_count
         <= float(acceptance["detector_maximum_drop_fraction"]),
         "semantic_accounting": int(semantic_state["consumed"])
         + int(semantic_state["dropped_overflow"])
@@ -1286,6 +1505,8 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
         },
         "latency_ms": latency_summary,
         "gpu_telemetry": gpu.summary(),
+        "runtime_telemetry": runtime_telemetry.summary(),
+        "process_cpu_seconds": time.process_time() - process_cpu_started,
         "process_peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
         "cuda_peak_memory_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
         "cuda_peak_memory_reserved_mib": torch.cuda.max_memory_reserved() / 2**20,
@@ -1335,6 +1556,12 @@ def run(args: argparse.Namespace, loaded: _LoadedModels | None = None) -> int:
         _artifact(fusion_path, "e15-fusion-frames", "application/x-ndjson", FUSION_SCHEMA),
         _artifact(world_path, "e15-world-state", "application/x-ndjson", WORLD_SCHEMA),
         _artifact(gpu_path, "worker-gpu-telemetry", "application/x-ndjson"),
+        _artifact(
+            runtime_path,
+            "worker-runtime-telemetry",
+            "application/x-ndjson",
+            "missioncore.worker-runtime-telemetry/v1",
+        ),
         _artifact(report_path, "e15-run-report", "application/json", REPORT_SCHEMA),
     ]
     _write_json(
@@ -1381,6 +1608,10 @@ def _persistent_run_arguments(
     request_id = request.get("request_id")
     output_name = request.get("output_name")
     token = request.get("token")
+    requested_duration = request.get(
+        "max_duration_seconds",
+        service_args.max_duration_seconds,
+    )
     if (
         not isinstance(request_id, str)
         or _CONTROL_ID.fullmatch(request_id) is None
@@ -1389,6 +1620,10 @@ def _persistent_run_arguments(
         or output_name in {".", ".."}
         or not isinstance(token, str)
         or not 40 <= len(token) <= 512
+        or not isinstance(requested_duration, int | float)
+        or isinstance(requested_duration, bool)
+        or not math.isfinite(float(requested_duration))
+        or not 5 <= float(requested_duration) <= float(service_args.max_duration_seconds)
     ):
         raise RuntimeError("persistent worker request contract is invalid")
     output_root = service_args.output_root.resolve(strict=True)
@@ -1401,6 +1636,7 @@ def _persistent_run_arguments(
         output=output,
         token=token,
         token_stdin=False,
+        max_duration_seconds=float(requested_duration),
     )
     values.pop("output_root", None)
     values.pop("listen_host", None)
@@ -1502,6 +1738,18 @@ def serve(args: argparse.Namespace) -> int:
                 )
             except Exception as exc:
                 state["failed_runs"] += 1
+                print(
+                    json.dumps(
+                        {
+                            "event": "persistent-worker-run-failed",
+                            "request_id": request_id,
+                            "error_type": type(exc).__name__,
+                            "error": str(exc)[:1000],
+                        },
+                        sort_keys=True,
+                    ),
+                    flush=True,
+                )
                 self._send(
                     422,
                     {

experiments/perception/worker/run_recorded_perception_epoch.py

@@ -246,7 +246,7 @@ def _write_png(path: Path, array: Any) -> None:
 
 def _percentiles(values: list[float]) -> dict[str, float]:
     if not values:
-        return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "max": 0.0}
+        return {"mean": 0.0, "p50": 0.0, "p95": 0.0, "p99": 0.0, "max": 0.0}
     ordered = sorted(values)
 
     def percentile(fraction: float) -> float:
@@ -257,6 +257,7 @@ def _percentiles(values: list[float]) -> dict[str, float]:
         "mean": round(statistics.fmean(values), 6),
         "p50": percentile(0.5),
         "p95": percentile(0.95),
+        "p99": percentile(0.99),
         "max": round(max(values), 6),
     }