feat: qualify bounded K1 surface shadow

2026-07-26 01:06:04 +03:00
parent 4c83e8a4e7
commit e6d5411bdd
8 changed files with 1370 additions and 14 deletions
@@ -0,0 +1,97 @@
+# LAB E27 — bounded K1 local-surface shadow qualification
+
+Date: 2026-07-26
+Status: **accepted for recorded-source-paced shadow diagnostic**
+Authority: diagnostic only; commands, navigation and safety acceptance disabled
+
+## 1. Question
+
+Can the accepted K1 rolling local-surface profile execute through a real
+bounded latest-wins worker loop at the recorded K1 source rate without queue
+growth, frame replacement or divergence from the immutable replay result?
+
+This lab does not ask whether the derived surface is ground truth or
+planner-ready. It qualifies execution semantics only.
+
+## 2. Fixed input
+
+- session: `20260720T065719Z_viewer_live`;
+- source pack:
+  `e10-lidar-pack-5da0396d32a27f9d1ca537cc2e8a371d386078d6f0dc71737b78620992af9625`;
+- source artifact SHA-256:
+  `72aa73340b20fcfaa21b330ef5b93b975c14a70e2cd16a57752d9952ff05ad9a`;
+- replay reference:
+  `k1-local-surface-628cd024775f02fea99765d1fb457efec2d5cc4d7371e56818dfe8e08c9b3b74`;
+- reference logical-content SHA-256:
+  `b89a92887cedace9d3eab3e1490697fb6bc7fc16a2d1887f3fff7cb4547ceb14`;
+- selection: source frames `0–150`, `14.995 s`, 151 timeline entries and
+  143 available LiDAR frames;
+- pace: recorded 1×;
+- work queue: latest-wins, capacity 2;
+- result ring: bounded to the 143-frame qualification selection.
+
+The source pack and replay derivative were opened read-only. No scanner,
+firmware, MQTT command or persistent reconstruction was changed.
+
+## 3. Implemented runtime boundary
+
+`missioncore.k1-local-surface-shadow-runtime/v1` accepts only an already
+decoded map-frame point cloud and a compatible `T_map_from_sensor` pose. It:
+
+1. copies and freezes the admitted point/pose pair;
+2. publishes work into a bounded latest-wins queue;
+3. runs the same robust rolling-cell and prior-only prediction profile used by
+   replay;
+4. retains only a bounded diagnostic result ring;
+5. reports observed surface, observed occupied-above-surface, negative
+   outliers, unverified step candidates, latency and freshness;
+6. explicitly keeps absence-of-points distinct from free space.
+
+The runtime has no command method and every result carries
+`commands_enabled=false` and `navigation_or_safety_accepted=false`.
+
+## 4. Result
+
+Result:
+`k1-local-surface-shadow-04f14d8c580f74cbd5b0a452867563ebc6b3ef93d872129e4918680932253ab7`
+
+| Metric | Result |
+| --- | ---: |
+| Published / consumed | 143 / 143 |
+| Latest-wins replacements | 0 |
+| Maximum queue depth | 1 / 2 |
+| Processing failures | 0 |
+| Processing p50 / p95 / max | 12.261 / 14.777 / 33.135 ms |
+| Result age p50 / p95 / max | 12.336 / 14.857 / 33.378 ms |
+| Replay state mismatches | 0 |
+| Point-class mismatches | 0 |
+| Step-candidate mismatches | 0 |
+| Maximum scalar delta | 0.0 |
+
+All 143 results were valid for this selection. Queue accounting closed exactly:
+`consumed + dropped_overflow = published`, final depth was zero and the worker
+thread stopped cleanly.
+
+## 5. Decision
+
+The execution gate passes. The current CPU geometric profile is comfortably
+inside the observed roughly 10 Hz K1 publication interval on this 15-second
+slice, remains bounded and reproduces replay exactly when no work is replaced.
+
+This result promotes the profile only from offline replay implementation to a
+recorded-source-paced shadow candidate. It does not promote:
+
+- the surface estimate to ground truth;
+- observed occupancy to free-space evidence;
+- step candidates to semantic curbs;
+- the worker result to navigation or safety authority;
+- the replay transport to a physical-live K1 gate.
+
+## 6. Next gate
+
+Connect the runtime to the existing authenticated external worker stream using
+an explicit bounded LiDAR↔pose binder, then repeat at least 15 seconds against
+a physical K1 acquisition. Measure source sequence gaps, pose-binding misses,
+queue replacements, result age, memory slope and recovery across reconnect.
+React remains a read-only status/review surface; it does not execute the
+algorithm.
@@ -0,0 +1,365 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from k1link.compute import (
+    DEFAULT_K1_LOCAL_SURFACE_PROFILE,
+    E10LidarFieldSource,
+    K1LocalSurfaceShadowInput,
+    K1LocalSurfaceShadowResult,
+    K1LocalSurfaceShadowRuntime,
+    K1LocalSurfaceV1,
+)
+from k1link.compute.lidar_local_surface import (
+    FRAME_FIT_FAILED,
+    FRAME_INSUFFICIENT_SURFACE,
+    FRAME_POSE_STALE,
+    FRAME_VALID,
+)
+
+QUALIFICATION_SCHEMA = "missioncore.k1-local-surface-shadow-qualification/v1"
+
+
+def _arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run the accepted K1 rolling-surface profile through a bounded "
+            "latest-wins replay shadow and compare processed frames to the "
+            "immutable replay derivative."
+        )
+    )
+    parser.add_argument("source_pack", type=Path)
+    parser.add_argument("reference_model", type=Path)
+    parser.add_argument("output_root", type=Path)
+    parser.add_argument("--start-frame", type=int, default=0)
+    parser.add_argument("--duration-seconds", type=float, default=15.0)
+    parser.add_argument("--pace-scale", type=float, default=1.0)
+    parser.add_argument("--queue-capacity", type=int, default=2)
+    return parser.parse_args()
+
+
+def _canonical_json(value: object) -> bytes:
+    return json.dumps(
+        value,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+        allow_nan=False,
+    ).encode()
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        while chunk := stream.read(1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _distribution(values: list[float]) -> dict[str, float | int | None]:
+    if not values:
+        return {
+            "sample_count": 0,
+            "minimum": None,
+            "mean": None,
+            "p50": None,
+            "p95": None,
+            "maximum": None,
+        }
+    array = np.asarray(values, dtype=np.float64)
+    if not np.isfinite(array).all():
+        raise RuntimeError("shadow qualification latency is invalid")
+    return {
+        "sample_count": int(array.shape[0]),
+        "minimum": float(np.min(array)),
+        "mean": float(np.mean(array)),
+        "p50": float(np.percentile(array, 50)),
+        "p95": float(np.percentile(array, 95)),
+        "maximum": float(np.max(array)),
+    }
+
+
+def _expected_state(model: K1LocalSurfaceV1, frame_index: int) -> str:
+    code = int(model.arrays["frame_failure_code"][frame_index])
+    return {
+        FRAME_VALID: "valid",
+        FRAME_POSE_STALE: "pose-stale",
+        FRAME_INSUFFICIENT_SURFACE: "insufficient-surface",
+        FRAME_FIT_FAILED: "fit-failed",
+    }[code]
+
+
+def _maximum_scalar_delta(
+    result: K1LocalSurfaceShadowResult,
+    model: K1LocalSurfaceV1,
+) -> float:
+    frame_index = result.frame_index
+    pairs = (
+        (result.sensor_height_m, "sensor_height_m"),
+        (result.slope_deg, "slope_deg"),
+        (result.roughness_m, "roughness_m"),
+        (result.confidence, "confidence"),
+        (result.surface_max_age_ms, "surface_max_age_ms"),
+    )
+    deltas = [
+        abs(float(value) - float(model.arrays[name][frame_index]))
+        for value, name in pairs
+        if value is not None
+    ]
+    return max(deltas, default=0.0)
+
+
+def _qualification(
+    source: E10LidarFieldSource,
+    model: K1LocalSurfaceV1,
+    *,
+    start_frame: int,
+    duration_seconds: float,
+    pace_scale: float,
+    queue_capacity: int,
+) -> dict[str, Any]:
+    if (
+        model.identity.get("source_pack_id") != source.pack_id
+        or model.identity.get("source_pack_identity_sha256")
+        != source.manifest.get("identity_sha256")
+        or model.identity.get("source_artifact_sha256")
+        != source.manifest.get("artifact", {}).get("sha256")
+        or model.identity.get("profile") != DEFAULT_K1_LOCAL_SURFACE_PROFILE.to_dict()
+    ):
+        raise RuntimeError("shadow qualification source/model binding is invalid")
+    if (
+        not 0 <= start_frame < source.frame_count
+        or not math.isfinite(duration_seconds)
+        or duration_seconds <= 0
+        or not math.isfinite(pace_scale)
+        or not 0 < pace_scale <= 10
+        or not 1 <= queue_capacity <= 8
+    ):
+        raise RuntimeError("shadow qualification selection is invalid")
+
+    arrays = source.arrays
+    session_times = arrays["session_seconds"]
+    start_seconds = float(session_times[start_frame])
+    selected = [
+        frame_index
+        for frame_index in range(start_frame, source.frame_count)
+        if float(session_times[frame_index]) - start_seconds <= duration_seconds
+    ]
+    if len(selected) < 2:
+        raise RuntimeError("shadow qualification selection is too short")
+    expected_available = [
+        frame_index for frame_index in selected if bool(arrays["sample_available"][frame_index])
+    ]
+    runtime = K1LocalSurfaceShadowRuntime(
+        f"{source.identity['session_id']}-qualification",
+        queue_capacity=queue_capacity,
+        result_capacity=min(256, max(1, len(expected_available))),
+    )
+    wall_started = time.perf_counter()
+    offsets = arrays["cloud_offsets"]
+    try:
+        for frame_index in selected:
+            release_at = (
+                wall_started + (float(session_times[frame_index]) - start_seconds) * pace_scale
+            )
+            remaining = release_at - time.perf_counter()
+            if remaining > 0:
+                time.sleep(remaining)
+            if not bool(arrays["sample_available"][frame_index]):
+                continue
+            start = int(offsets[frame_index])
+            end = int(offsets[frame_index + 1])
+            runtime.publish(
+                K1LocalSurfaceShadowInput(
+                    frame_index=frame_index,
+                    source_frame_index=int(arrays["source_frame_indices"][frame_index]),
+                    session_seconds=float(session_times[frame_index]),
+                    pose_binding_age_ms=abs(float(arrays["pose_point_delta_ms"][frame_index])),
+                    points_map=np.asarray(
+                        arrays["cloud_points_map"][start:end],
+                        dtype=np.float64,
+                    ),
+                    position_map=np.asarray(
+                        arrays["pose_positions_map"][frame_index],
+                        dtype=np.float64,
+                    ),
+                    published_monotonic_ns=time.monotonic_ns(),
+                )
+            )
+        runtime.close(timeout_seconds=30.0)
+        results = runtime.results()
+        runtime_snapshot = runtime.snapshot()
+    finally:
+        runtime.close()
+
+    state_mismatches = 0
+    point_class_mismatches = 0
+    step_candidate_mismatches = 0
+    maximum_scalar_delta = 0.0
+    for result in results:
+        frame_index = result.frame_index
+        state_mismatches += result.state != _expected_state(model, frame_index)
+        if result.valid:
+            start = int(offsets[frame_index])
+            end = int(offsets[frame_index + 1])
+            point_class_mismatches += int(
+                np.count_nonzero(result.point_class != model.arrays["point_class"][start:end])
+            )
+            step_candidate_mismatches += int(
+                np.count_nonzero(
+                    result.point_step_candidate != model.arrays["point_step_candidate"][start:end]
+                )
+            )
+            maximum_scalar_delta = max(
+                maximum_scalar_delta,
+                _maximum_scalar_delta(result, model),
+            )
+    queue = runtime_snapshot["queue"]
+    accepted = (
+        int(queue["maximum_depth"]) <= queue_capacity
+        and int(queue["dropped_overflow"]) == 0
+        and int(queue["consumed"]) == len(expected_available)
+        and len(results) == len(expected_available)
+        and int(runtime_snapshot["results"]["failed"]) == 0
+        and state_mismatches == 0
+        and point_class_mismatches == 0
+        and step_candidate_mismatches == 0
+        and maximum_scalar_delta <= 1e-9
+    )
+    return {
+        "schema_version": QUALIFICATION_SCHEMA,
+        "identity": {
+            "source_pack_id": source.pack_id,
+            "source_pack_identity_sha256": source.manifest["identity_sha256"],
+            "source_artifact_sha256": source.manifest["artifact"]["sha256"],
+            "reference_model_id": model.model_id,
+            "reference_logical_content_sha256": model.identity["logical_content_sha256"],
+            "session_id": source.identity["session_id"],
+            "profile": DEFAULT_K1_LOCAL_SURFACE_PROFILE.to_dict(),
+            "selection": {
+                "start_frame": start_frame,
+                "end_frame": selected[-1],
+                "source_duration_seconds": (float(session_times[selected[-1]]) - start_seconds),
+                "selected_frames": len(selected),
+                "available_frames": len(expected_available),
+            },
+            "runtime": {
+                "queue_policy": "bounded-latest-wins",
+                "queue_capacity": queue_capacity,
+                "result_capacity": min(
+                    256,
+                    max(1, len(expected_available)),
+                ),
+                "pace_scale": pace_scale,
+            },
+            "producer_sha256": _sha256(Path(__file__).resolve(strict=True)),
+        },
+        "state": "accepted" if accepted else "rejected",
+        "accepted": accepted,
+        "ground_truth": False,
+        "metrics": {
+            "wall_elapsed_seconds": time.perf_counter() - wall_started,
+            "queue": queue,
+            "result_states": runtime_snapshot["results"]["state_counts"],
+            "processing_ms": _distribution([result.processing_ms for result in results]),
+            "result_age_ms": _distribution([result.result_age_ms for result in results]),
+            "replay_parity": {
+                "compared_frames": len(results),
+                "state_mismatches": state_mismatches,
+                "point_class_mismatches": point_class_mismatches,
+                "step_candidate_mismatches": step_candidate_mismatches,
+                "maximum_scalar_delta": maximum_scalar_delta,
+            },
+        },
+        "acceptance": {
+            "queue_bounded": int(queue["maximum_depth"]) <= queue_capacity,
+            "zero_latest_wins_replacements": int(queue["dropped_overflow"]) == 0,
+            "zero_processing_failures": (int(runtime_snapshot["results"]["failed"]) == 0),
+            "complete_processed_accounting": (
+                int(queue["consumed"]) == len(expected_available)
+                and len(results) == len(expected_available)
+            ),
+            "replay_state_parity": state_mismatches == 0,
+            "replay_point_class_parity": point_class_mismatches == 0,
+            "replay_step_candidate_parity": step_candidate_mismatches == 0,
+            "replay_scalar_parity": maximum_scalar_delta <= 1e-9,
+            "navigation_or_safety_accepted": False,
+        },
+        "occupancy_policy": {
+            "absence_of_points_means_free": False,
+            "unknown_is_traversable": False,
+        },
+        "authority": {
+            "commands_enabled": False,
+            "navigation_or_safety_accepted": False,
+        },
+    }
+
+
+def main() -> int:
+    arguments = _arguments()
+    source = E10LidarFieldSource(arguments.source_pack)
+    model = K1LocalSurfaceV1(arguments.reference_model)
+    try:
+        report = _qualification(
+            source,
+            model,
+            start_frame=arguments.start_frame,
+            duration_seconds=arguments.duration_seconds,
+            pace_scale=arguments.pace_scale,
+            queue_capacity=arguments.queue_capacity,
+        )
+    finally:
+        model.close()
+        source.close()
+    report_sha256 = hashlib.sha256(_canonical_json(report)).hexdigest()
+    result_id = f"k1-local-surface-shadow-{report_sha256}"
+    report["result_id"] = result_id
+    report["report_sha256"] = report_sha256
+    report["created_at_utc"] = datetime.now(UTC).isoformat()
+    output_root = arguments.output_root.expanduser().resolve()
+    output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
+    output = output_root / result_id
+    if output.exists():
+        raise RuntimeError("shadow qualification result already exists")
+    staging = output_root / f".{result_id}.{os.getpid()}.incomplete"
+    staging.mkdir(mode=0o700, exist_ok=False)
+    try:
+        report_path = staging / "report.json"
+        report_path.write_bytes(_canonical_json(report))
+        os.replace(staging, output)
+    except BaseException:
+        if staging.exists():
+            for path in staging.iterdir():
+                path.unlink()
+            staging.rmdir()
+        raise
+    print(
+        json.dumps(
+            {
+                "result_id": result_id,
+                "output": str(output),
+                "state": report["state"],
+                "metrics": report["metrics"],
+                "authority": report["authority"],
+            },
+            ensure_ascii=False,
+            indent=2,
+        )
+    )
+    return 0 if report["accepted"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())