SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wip(k1): checkpoint connection recovery rewrite

Browse Source 
Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
DCCONSTRUCTIONS
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/worker/run_e15_shadow_inference.py
+52 -100
View File
@@ -254,13 +254,11 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
}
if (
not isinstance(replay_source, dict)
or replay_source.get("session_id")
!= "20260720T065719Z_viewer_live"
or replay_source.get("session_id") != "20260720T065719Z_viewer_live"
or replay_source.get("display_name") != "RAVNOVES00"
or replay_source.get("selection") != "complete-recording"
or float(replay_source.get("speed", 0)) != 1.0
or float(replay_source.get("minimum_source_span_seconds", 0))
< 450
or float(replay_source.get("minimum_source_span_seconds", 0)) < 450
or replay_source.get("look_ahead") is not False
or any(
not isinstance(replay_source.get(key), int)
@@ -269,9 +267,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
for key, expected in replay_integer_contract.items()
)
):
raise RuntimeError(
"LAB E28 complete-recording worker replay contract is invalid"
)
raise RuntimeError("LAB E28 complete-recording worker replay contract is invalid")
elif replay_source is not None:
raise RuntimeError("LAB E15 non-replay profile carries replay source state")
if local_surface is not None:
@@ -280,9 +276,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
)
local_acceptance = (
local_surface.get("acceptance")
if isinstance(local_surface, dict)
else None
local_surface.get("acceptance") if isinstance(local_surface, dict) else None
)
expected_profile_sha256 = hashlib.sha256(
canonical_json(DEFAULT_K1_LOCAL_SURFACE_PROFILE.to_dict())
@@ -293,29 +287,20 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
"maximum_runtime_drop_fraction",
)
point_capacity = (
local_surface.get("point_queue_capacity")
if isinstance(local_surface, dict)
else None
local_surface.get("point_queue_capacity") if isinstance(local_surface, dict) else None
)
pose_capacity = (
local_surface.get("pose_buffer_capacity")
if isinstance(local_surface, dict)
else None
local_surface.get("pose_buffer_capacity") if isinstance(local_surface, dict) else None
)
result_capacity = (
local_surface.get("result_capacity")
if isinstance(local_surface, dict)
else None
local_surface.get("result_capacity") if isinstance(local_surface, dict) else None
)
if (
profile.get("mode")
not in {"worker-replay-gate", "physical-shadow-gate"}
profile.get("mode") not in {"worker-replay-gate", "physical-shadow-gate"}
or not isinstance(local_surface, dict)
or local_surface.get("enabled") is not True
or local_surface.get("profile_id")
!= DEFAULT_K1_LOCAL_SURFACE_PROFILE.profile_id
or local_surface.get("profile_sha256")
!= expected_profile_sha256
or local_surface.get("profile_id") != DEFAULT_K1_LOCAL_SURFACE_PROFILE.profile_id
or local_surface.get("profile_sha256") != expected_profile_sha256
or not isinstance(point_capacity, int)
or isinstance(point_capacity, bool)
or point_capacity not in range(1, 9)
@@ -328,27 +313,16 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]:
or not 0
<= float(local_surface.get("future_pose_wait_ms", -1))
<= DEFAULT_K1_LOCAL_SURFACE_PROFILE.maximum_pose_binding_ms
or not 0.1
<= float(local_surface.get("retention_seconds", 0))
<= 30
or not 0.1 <= float(local_surface.get("retention_seconds", 0)) <= 30
or float(temporal.get("maximum_pose_point_delta_ms", 0))
!= DEFAULT_K1_LOCAL_SURFACE_PROFILE.maximum_pose_binding_ms
or not isinstance(local_acceptance, dict)
or int(local_acceptance.get("minimum_bound_frames", 0)) < 2
or any(
not 0 <= float(local_acceptance.get(key, -1)) <= 1
for key in local_fractions
)
or float(
local_acceptance.get("maximum_p95_result_age_ms", 0)
)
<= 0
or float(local_acceptance.get("minimum_effective_fps", 0))
<= 0
or any(not 0 <= float(local_acceptance.get(key, -1)) <= 1 for key in local_fractions)
or float(local_acceptance.get("maximum_p95_result_age_ms", 0)) <= 0
or float(local_acceptance.get("minimum_effective_fps", 0)) <= 0
):
raise RuntimeError(
"LAB E28 worker local-surface profile contract is invalid"
)
raise RuntimeError("LAB E28 worker local-surface profile contract is invalid")
fractions = (
"detector_maximum_drop_fraction",
"semantic_maximum_drop_fraction",
@@ -429,64 +403,45 @@ def _local_surface_acceptance_checks(
return {
"local_surface_session_initialized": bool(runtime),
"local_surface_closed": snapshot.get("closed") is True
and runtime.get("closed") is True,
"local_surface_closed": snapshot.get("closed") is True and runtime.get("closed") is True,
"local_surface_minimum_bound_frames": point_bound
>= int(acceptance["minimum_bound_frames"]),
"local_surface_binder_accounting": point_bound
+ point_missed
+ point_dropped
+ point_depth
"local_surface_binder_accounting": point_bound + point_missed + point_dropped + point_depth
== point_published,
"local_surface_binder_to_runtime_accounting": point_bound
== runtime_published,
"local_surface_binder_to_runtime_accounting": point_bound == runtime_published,
"local_surface_point_buffer_bound": (
int(points.get("capacity", 0)) == int(config["point_queue_capacity"])
and int(points.get("maximum_depth", 0))
<= int(points.get("capacity", 0))
and int(points.get("maximum_depth", 0)) <= int(points.get("capacity", 0))
and point_depth == 0
),
"local_surface_pose_buffer_bound": (
int(poses.get("capacity", 0)) == int(config["pose_buffer_capacity"])
and int(poses.get("maximum_depth", 0))
<= int(poses.get("capacity", 0))
and int(poses.get("maximum_depth", 0)) <= int(poses.get("capacity", 0))
),
"local_surface_maximum_pose_miss_fraction": point_missed
/ max(1, point_published)
"local_surface_maximum_pose_miss_fraction": point_missed / max(1, point_published)
<= float(acceptance["maximum_pose_miss_fraction"]),
"local_surface_maximum_point_drop_fraction": point_dropped
/ max(1, point_published)
"local_surface_maximum_point_drop_fraction": point_dropped / max(1, point_published)
<= float(acceptance["maximum_point_drop_fraction"]),
"local_surface_runtime_accounting": runtime_consumed
+ runtime_dropped
+ runtime_depth
"local_surface_runtime_accounting": runtime_consumed + runtime_dropped + runtime_depth
== runtime_published,
"local_surface_runtime_result_accounting": result_published
+ result_failed
"local_surface_runtime_result_accounting": result_published + result_failed
== runtime_consumed,
"local_surface_runtime_queue_bound": (
int(queue_state.get("capacity", 0)) == int(config["point_queue_capacity"])
and int(queue_state.get("maximum_depth", 0))
<= int(queue_state.get("capacity", 0))
and int(queue_state.get("maximum_depth", 0)) <= int(queue_state.get("capacity", 0))
and runtime_depth == 0
),
"local_surface_maximum_runtime_drop_fraction": runtime_dropped
/ max(1, runtime_published)
"local_surface_maximum_runtime_drop_fraction": runtime_dropped / max(1, runtime_published)
<= float(acceptance["maximum_runtime_drop_fraction"]),
"local_surface_minimum_effective_fps": float(
delivery.get("effective_fps", 0)
)
"local_surface_minimum_effective_fps": float(delivery.get("effective_fps", 0))
>= float(acceptance["minimum_effective_fps"]),
"local_surface_zero_runtime_failures": result_failed == 0,
"local_surface_maximum_p95_result_age_ms": (
isinstance(p95_result_age, (int, float))
and not isinstance(p95_result_age, bool)
and float(p95_result_age)
<= float(acceptance["maximum_p95_result_age_ms"])
),
"local_surface_profile_pinned": (
runtime_profile.get("profile_id") == config["profile_id"]
and float(p95_result_age) <= float(acceptance["maximum_p95_result_age_ms"])
),
"local_surface_profile_pinned": (runtime_profile.get("profile_id") == config["profile_id"]),
"local_surface_shadow_authority_only": (
snapshot.get("authority")
== {
@@ -622,6 +577,7 @@ class _TransportState:
camera_sequence_gaps: int = 0
last_camera_source_sequence: int | None = None
session_id: str | None = None
session_generation: int | None = None
session_end_seen: bool = False
timed_out: bool = False
results_published: int = 0
@@ -956,8 +912,7 @@ class _StageExecutionTelemetry:
self._last_frame_by_stage.get(stage_id),
)
for stage_id in self._stage_ids
if stage_id in self._native_started
and stage_id not in self._native_failed
if stage_id in self._native_started and stage_id not in self._native_failed
]
for stage_id, elapsed_seconds, activations, frame_index in rows:
self._emit_native(
@@ -1012,9 +967,7 @@ class _StageExecutionTelemetry:
"elapsed_seconds": round(elapsed[stage_id], 6),
"activations": self._activations[stage_id],
"share_percent": (
round(elapsed[stage_id] / total * 100, 6)
if total > 0
else None
round(elapsed[stage_id] / total * 100, 6) if total > 0 else None
),
}
for stage_id in self._stage_ids
@@ -1127,11 +1080,20 @@ def _receiver(
state.first_ingress_sequence = sequence
state.last_ingress_sequence = sequence
session_id = str(header["session_id"])
session_generation_value = header["session_generation"]
if (
not isinstance(session_generation_value, int)
or isinstance(session_generation_value, bool)
or session_generation_value < 1
):
raise ShadowRuntimeError("shadow session generation is invalid")
session_generation = session_generation_value
if state.session_id is None:
state.session_id = session_id
state.session_generation = session_generation
if local_surface is not None:
local_surface.begin_session(session_id)
elif state.session_id != session_id:
elif state.session_id != session_id or state.session_generation != session_generation:
raise ShadowRuntimeError("shadow session identity changed")
modality = str(header["modality"])
state.counts[modality] += 1
@@ -1272,9 +1234,7 @@ def _common(args: argparse.Namespace) -> dict[str, Any]:
"k1link/ground_segmentation.py",
}
if not required_surface_sources <= worker_sources:
raise RuntimeError(
"LAB E28 worker package lacks local-surface runtime"
)
raise RuntimeError("LAB E28 worker package lacks local-surface runtime")
stability = None
stability_sha256 = None
if args.stability_profile is not None:
@@ -1400,11 +1360,7 @@ def run(
if not token or len(token) < 40:
raise RuntimeError("LAB E15 shadow token is missing")
stage_telemetry = (
runtime_state.get("_stage_telemetry")
if runtime_state is not None
else None
)
stage_telemetry = runtime_state.get("_stage_telemetry") if runtime_state is not None else None
if not isinstance(stage_telemetry, _StageExecutionTelemetry):
stage_telemetry = _StageExecutionTelemetry()
if runtime_state is not None:
@@ -1461,9 +1417,7 @@ def run(
local_surface = K1LocalSurfaceShadowCoordinator(
point_capacity=int(local_surface_config["point_queue_capacity"]),
pose_capacity=int(local_surface_config["pose_buffer_capacity"]),
future_pose_wait_ms=float(
local_surface_config["future_pose_wait_ms"]
),
future_pose_wait_ms=float(local_surface_config["future_pose_wait_ms"]),
retention_seconds=float(local_surface_config["retention_seconds"]),
result_capacity=int(local_surface_config["result_capacity"]),
)
@@ -1890,7 +1844,11 @@ def run(
optimize=False,
)
with stage_telemetry.measure("result-publication", envelope.frame_index):
if transport.session_id is None or transport.session_generation is None:
raise ShadowRuntimeError("shadow result session identity is unavailable")
live_result = encode_live_perception_result(
session_id=transport.session_id,
session_generation=transport.session_generation,
frame_index=envelope.frame_index,
source_frame_index=int(envelope.timeline["source_frame_index"]),
session_seconds=frame_seconds,
@@ -1969,9 +1927,7 @@ def run(
temporal_semantic_summary = (
None if semantic_stabilizer is None else semantic_stabilizer.snapshot()
)
local_surface_snapshot = (
None if local_surface is None else local_surface.snapshot()
)
local_surface_snapshot = None if local_surface is None else local_surface.snapshot()
acceptance = live["acceptance"]
checks = {
"minimum_camera_frames": decoded_frame_count >= int(acceptance["minimum_camera_frames"]),
@@ -2356,9 +2312,7 @@ def _persistent_run_telemetry_identity(
if isinstance(stability, dict) and isinstance(stability.get("profile_id"), str)
else "lab-e15-shadow-inference-v1"
)
method_id = (
INLINE_TEMPORAL_PIPELINE_ID if isinstance(stability, dict) else PIPELINE_ID
)
method_id = INLINE_TEMPORAL_PIPELINE_ID if isinstance(stability, dict) else PIPELINE_ID
return PipelineTelemetryIdentity(
contour_id=telemetry.get("contour_id"),
agent_id=telemetry.get("agent_id"),
@@ -2545,9 +2499,7 @@ def serve(args: argparse.Namespace) -> int:
state["last_run_outcome"] = {
"request_id": request_id,
"state": "failed",
"duration_ms": (
round(duration_ms, 6) if duration_ms is not None else None
),
"duration_ms": (round(duration_ms, 6) if duration_ms is not None else None),
"exit_code": None,
"error_type": type(exc).__name__,
}