SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wip(k1): checkpoint connection recovery rewrite

Browse Source 
Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
DCCONSTRUCTIONS
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_rerun_bridge.py
+176 -10
View File
@@ -1,9 +1,11 @@
from __future__ import annotations
import errno
import json
import logging
import socket
import struct
import sys
import threading
import time
from pathlib import Path
@@ -25,6 +27,8 @@ from k1link.viewer.rerun_bridge import (
_select_available_grpc_port,
)
rerun_bridge_module = sys.modules["k1link.viewer.rerun_bridge"]
class FakeRecording:
def __init__(self) -> None:
@@ -65,11 +69,73 @@ class DisconnectFailureRecording(FakeRecording):
raise RuntimeError("synthetic disconnect failure")
@pytest.fixture
def socket_free_rerun_port_selector(monkeypatch: pytest.MonkeyPatch) -> None:
"""Keep FakeRecording tests independent of host TCP bind permission."""
monkeypatch.setattr(
rerun_bridge_module,
"_select_available_grpc_port",
lambda preferred_port, **_kwargs: preferred_port,
)
def test_runtime_owner_recovery_wake_is_generation_fenced_and_coalesced() -> None:
runtime = VisualizationRuntime(normalizer=normalize_k1_message)
with runtime._lock: # noqa: SLF001 - bounded producer-state unit seam
runtime._producer_generation = 7 # noqa: SLF001
runtime._source_mode = "live" # noqa: SLF001
runtime._phase = "live" # noqa: SLF001
runtime._source_ready = True # noqa: SLF001
runtime._connection_recovery_enabled = True # noqa: SLF001
assert (
runtime.request_connection_recovery(
"camera-source-ended",
expected_generation=6,
)
is False
)
assert runtime.request_connection_recovery(
"camera-source-ended",
expected_generation=7,
)
assert runtime.request_connection_recovery(
"mqtt_network_loop_failed",
expected_generation=7,
)
snapshot = runtime.snapshot()
assert snapshot["phase"] == "reconnecting"
assert snapshot["connection_recovery"]["reason_code"] == "camera-source-ended"
assert runtime._consume_connection_recovery_request(generation=6) is None # noqa: SLF001
assert ( # noqa: SLF001
runtime._consume_connection_recovery_request(generation=7)
== "camera-source-ended"
)
assert runtime._consume_connection_recovery_request(generation=7) is None # noqa: SLF001
with runtime._lock: # noqa: SLF001
runtime._phase = "live" # noqa: SLF001
runtime._source_ready = True # noqa: SLF001
assert runtime.request_connection_recovery(
"host-route-unavailable",
expected_generation=7,
)
runtime.stop()
assert runtime.snapshot()["phase"] == "idle"
assert runtime._consume_connection_recovery_request(generation=7) is None # noqa: SLF001
def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer(
caplog: pytest.LogCaptureFixture,
) -> None:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as occupied:
occupied.bind(("0.0.0.0", 0))
try:
occupied.bind(("0.0.0.0", 0))
except OSError as exc:
if exc.errno in {errno.EACCES, errno.EPERM}:
pytest.skip(f"host sandbox denies TCP bind: errno={exc.errno}")
raise
occupied.listen()
preferred_port = int(occupied.getsockname()[1])
@@ -92,6 +158,82 @@ def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer(
bridge.close()
def test_rerun_port_selection_retries_only_address_in_use(
monkeypatch: pytest.MonkeyPatch,
) -> None:
attempts: list[int] = []
class Probe:
def __enter__(self) -> Probe:
return self
def __exit__(self, *_args: object) -> None:
return
def bind(self, address: tuple[str, int]) -> None:
attempts.append(address[1])
if len(attempts) < 3:
raise OSError(errno.EADDRINUSE, "synthetic address in use")
monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe())
assert _select_available_grpc_port(9876, search_span=4) == 9878
assert attempts == [9876, 9877, 9878]
@pytest.mark.parametrize("error_number", [errno.EPERM, errno.EACCES])
def test_rerun_port_selection_reports_permission_denial_immediately(
monkeypatch: pytest.MonkeyPatch,
error_number: int,
) -> None:
attempts: list[int] = []
class Probe:
def __enter__(self) -> Probe:
return self
def __exit__(self, *_args: object) -> None:
return
def bind(self, address: tuple[str, int]) -> None:
attempts.append(address[1])
raise OSError(error_number, "synthetic permission denial")
monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe())
with pytest.raises(PermissionError, match="Permission denied.*9876") as error:
_select_available_grpc_port(9876, search_span=4)
assert error.value.errno == error_number
assert attempts == [9876]
@pytest.mark.parametrize("error_number", [errno.EADDRNOTAVAIL, errno.EIO])
def test_rerun_port_selection_does_not_misclassify_unexpected_socket_errors(
monkeypatch: pytest.MonkeyPatch,
error_number: int,
) -> None:
attempts: list[int] = []
class Probe:
def __enter__(self) -> Probe:
return self
def __exit__(self, *_args: object) -> None:
return
def bind(self, address: tuple[str, int]) -> None:
attempts.append(address[1])
raise OSError(error_number, "synthetic unexpected bind failure")
monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe())
with pytest.raises(RuntimeError, match="Could not probe.*9876"):
_select_available_grpc_port(9876, search_span=4)
assert attempts == [9876]
def _message(
topic: str,
payload: bytes,
@@ -129,7 +271,9 @@ def _envelope(
return envelope
def test_legacy_points_and_pose_are_logged_to_rerun() -> None:
def test_legacy_points_and_pose_are_logged_to_rerun(
socket_free_rerun_port_selector: None,
) -> None:
recording = FakeRecording()
bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type]
@@ -169,7 +313,9 @@ def test_live_blueprint_follows_stream_time_without_frontend_cursor_writes() ->
assert panel.state == "hidden"
def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None:
def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid(
socket_free_rerun_port_selector: None,
) -> None:
recording = FakeRecording()
bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type]
mask = np.zeros((600, 800), dtype=np.uint8)
@@ -177,6 +323,8 @@ def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None:
bridge.process_perception(
LivePerceptionResultFrame(
session_id="test-live-perception-session",
session_generation=1,
frame_index=3,
source_frame_index=30,
session_seconds=1.0,
@@ -206,7 +354,9 @@ def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None:
assert "/world/perception/boxes3d" in paths
def test_constructor_disconnects_recording_after_partial_setup_failure() -> None:
def test_constructor_disconnects_recording_after_partial_setup_failure(
socket_free_rerun_port_selector: None,
) -> None:
recording = BlueprintFailureRecording()
with pytest.raises(RuntimeError, match="synthetic blueprint failure"):
@@ -215,7 +365,9 @@ def test_constructor_disconnects_recording_after_partial_setup_failure() -> None
assert recording.disconnected is True
def test_fast_replay_trajectory_sampling_uses_source_time() -> None:
def test_fast_replay_trajectory_sampling_uses_source_time(
socket_free_rerun_port_selector: None,
) -> None:
recording = FakeRecording()
bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type]
base_time_ns = 1_784_124_315_000_000_000
@@ -245,7 +397,9 @@ def test_fast_replay_trajectory_sampling_uses_source_time() -> None:
bridge.close()
def test_bad_frame_is_rejected_before_rerun_without_publishing() -> None:
def test_bad_frame_is_rejected_before_rerun_without_publishing(
socket_free_rerun_port_selector: None,
) -> None:
recording = FakeRecording()
bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type]
@@ -296,7 +450,10 @@ def test_palettes_are_deterministic_and_custom_color_is_exact() -> None:
assert custom_over_rgb.tolist() == [[16, 32, 48], [16, 32, 48]]
def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) -> None:
def test_runtime_owns_fresh_bridge_for_each_sequential_session(
tmp_path: Path,
socket_free_rerun_port_selector: None,
) -> None:
capture = tmp_path / "mqtt.raw.k1mqtt"
point_topic = "RealtimePointcloud"
pose_topic = "RealtimePath"
@@ -380,7 +537,10 @@ def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) -
assert runtime.snapshot()["rerun_grpc_url"] is None
def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Path) -> None:
def test_runtime_reports_bridge_close_failure_instead_of_false_idle(
tmp_path: Path,
socket_free_rerun_port_selector: None,
) -> None:
capture = tmp_path / "mqtt.raw.k1mqtt"
topic = b"RealtimePointcloud"
payload = struct.pack("<III", 16, 0, 0) + struct.pack(
@@ -423,7 +583,10 @@ def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Pa
runtime.close()
def test_close_during_blocked_factory_closes_the_late_bridge(tmp_path: Path) -> None:
def test_close_during_blocked_factory_closes_the_late_bridge(
tmp_path: Path,
socket_free_rerun_port_selector: None,
) -> None:
capture = tmp_path / "mqtt.raw.k1mqtt"
topic = b"RealtimePointcloud"
payload = struct.pack("<III", 16, 0, 0) + struct.pack(
@@ -476,7 +639,10 @@ def test_close_during_blocked_factory_closes_the_late_bridge(tmp_path: Path) ->
runtime.start_replay(capture, speed=0.0)
def test_stop_fails_closed_when_runtime_thread_misses_deadline(tmp_path: Path) -> None:
def test_stop_fails_closed_when_runtime_thread_misses_deadline(
tmp_path: Path,
socket_free_rerun_port_selector: None,
) -> None:
capture = tmp_path / "mqtt.raw.k1mqtt"
topic = b"RealtimePointcloud"
payload = struct.pack("<III", 16, 0, 0) + struct.pack(