wip(k1): checkpoint connection recovery rewrite

Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
@@ -82,6 +82,99 @@ class FakeClient:
        return mqtt.MQTT_ERR_SUCCESS


+class ConnectionLostAfterMessageClient(FakeClient):
+    """Emit one complete subscription/message cycle, then lose the socket."""
+
+    def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
+        if self._step < 3:  # noqa: SLF001 - deterministic MQTT test double
+            return super().loop(timeout)
+        assert timeout > 0
+        self._step += 1  # noqa: SLF001
+        return mqtt.MQTT_ERR_CONN_LOST
+
+
+class SubscriptionUnavailableClient(FakeClient):
+    """Accept TCP/CONNACK, then fail the transient recovery subscription."""
+
+    def subscribe(self, topics: Any) -> tuple[mqtt.MQTTErrorCode, int]:
+        self.subscribe_calls.append(topics)
+        return mqtt.MQTT_ERR_NO_CONN, 0
+
+
+class SubscribedWithoutPointCloudClient(FakeClient):
+    """Reach SUBACK, emit only non-PCL reports, then lose the socket."""
+
+    def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
+        if self._step < 2:  # noqa: SLF001 - deterministic MQTT test double
+            return super().loop(timeout)
+        assert timeout > 0
+        self._step += 1  # noqa: SLF001
+        if self._step == 3:  # noqa: SLF001
+            assert self.on_message is not None
+            message = mqtt.MQTTMessage(topic=b"lixel/application/report/heartbeat")
+            message.payload = b"fresh-heartbeat"
+            message.qos = 0
+            self.on_message(self, None, message)
+            return mqtt.MQTT_ERR_SUCCESS
+        if self._step == 4:  # noqa: SLF001
+            assert self.on_message is not None
+            message = mqtt.MQTTMessage(topic=b"lixel/application/report/device_status")
+            message.payload = b"fresh-status"
+            message.qos = 0
+            self.on_message(self, None, message)
+            return mqtt.MQTT_ERR_SUCCESS
+        if self._step == 5:  # noqa: SLF001
+            assert self.on_message is not None
+            message = mqtt.MQTTMessage(topic=b"lixel/application/report/lio_pose")
+            message.payload = b"fresh-pose"
+            message.qos = 0
+            self.on_message(self, None, message)
+            return mqtt.MQTT_ERR_SUCCESS
+        return mqtt.MQTT_ERR_CONN_LOST
+
+
+class SpatialSequenceClient(FakeClient):
+    def __init__(self, messages: list[tuple[str, bytes, bool]]) -> None:
+        super().__init__()
+        self._messages = messages
+
+    def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
+        if self._step < 2:  # noqa: SLF001 - deterministic MQTT test double
+            return super().loop(timeout)
+        assert timeout > 0
+        self._step += 1  # noqa: SLF001
+        message_index = self._step - 3  # noqa: SLF001
+        if message_index < len(self._messages):
+            assert self.on_message is not None
+            topic, payload, retain = self._messages[message_index]
+            message = mqtt.MQTTMessage(topic=topic.encode())
+            message.payload = payload
+            message.qos = 0
+            message.retain = retain
+            self.on_message(self, None, message)
+            return mqtt.MQTT_ERR_SUCCESS
+        raise KeyboardInterrupt
+
+
+class LateOldClientCallbackRecoveryClient(SpatialSequenceClient):
+    """Replay one late callback from the retired client before fresh data."""
+
+    def __init__(self, old_client: FakeClient) -> None:
+        super().__init__([("RealtimePointcloud", b"fresh-new-client-pcl", False)])
+        self._old_client = old_client
+
+    def loop(self, timeout: float) -> mqtt.MQTTErrorCode:
+        if self._step < 2:  # noqa: SLF001 - deterministic MQTT test double
+            return super().loop(timeout)
+        if self._step == 2:  # noqa: SLF001
+            assert self._old_client.on_message is not None
+            late = mqtt.MQTTMessage(topic=b"RealtimePointcloud")
+            late.payload = b"late-old-client-pcl"
+            late.qos = 0
+            self._old_client.on_message(self._old_client, None, late)
+        return super().loop(timeout)
+
+
@pytest.mark.parametrize("address", ["10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.4.2"])
 def test_validate_private_ipv4_accepts_only_rfc1918(address: str) -> None:
    assert validate_private_ipv4(address) == address
@@ -96,6 +189,25 @@ def test_validate_private_ipv4_rejects_other_targets(address: str) -> None:
        validate_private_ipv4(address)


+@pytest.mark.parametrize(
+    ("attempt", "expected"),
+    [
+        (1, 0.5),
+        (2, 1.0),
+        (3, 2.0),
+        (4, 4.0),
+        (5, 5.0),
+        (1025, 5.0),
+        (10**100, 5.0),
+    ],
+)
+def test_recovery_backoff_saturates_without_unbounded_exponentiation(
+    attempt: int,
+    expected: float,
+) -> None:
+    assert capture_module._recovery_backoff_seconds(attempt) == expected  # noqa: SLF001
+
+
 def test_capture_writes_verifiable_frames_metadata_and_summary(tmp_path: Path) -> None:
    fake = FakeClient()
    observed = []
@@ -176,6 +288,7 @@ def test_capture_writes_verifiable_frames_metadata_and_summary(tmp_path: Path) -
    for artifact_name in (
        "mqtt.raw.k1mqtt",
        "mqtt.metadata.jsonl",
+        "mqtt.recovery.jsonl",
        "mqtt.timeline.origin.json",
        "mqtt.timeline.json",
        "mqtt.summary.json",
@@ -264,6 +377,360 @@ def test_capture_can_be_stopped_by_owner_without_losing_artifacts(tmp_path: Path
    assert list(iter_capture_frames(tmp_path / "capture" / "mqtt.raw.k1mqtt"))[0].payload


+def test_guarded_recovery_keeps_one_evidence_writer_and_never_publishes(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    second = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery")
+    clients = iter((first, second))
+    losses: list[str] = []
+    attempts: list[int] = []
+    candidates: list[tuple[int, int]] = []
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "recovering-capture",
+        duration_seconds=30,
+        on_connection_lost=losses.append,
+        recover_connection=lambda attempt: attempts.append(attempt) or "resume",
+        on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append(
+            (attempt, sequence)
+        ),
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    frames = list(iter_capture_frames(tmp_path / "recovering-capture" / "mqtt.raw.k1mqtt"))
+    assert [frame.payload for frame in frames] == [b"before-loss", b"after-recovery"]
+    assert losses and "network loop failed" in losses[0]
+    assert attempts == [1]
+    assert candidates == [(1, 2)]
+    assert summary["reconnect_enabled"] is True
+    assert summary["recovery_attempts"] == 1
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+    assert summary["recovery_blocked"] is False
+    assert summary["publishing_enabled"] is False
+    assert first.subscribe_calls and second.subscribe_calls
+
+
+def test_recovery_success_requires_one_exact_post_publish_confirmation(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    second = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery")
+    clients = iter((first, second))
+    confirmers: list[Callable[[int], bool]] = []
+    confirmation_results: list[bool] = []
+
+    def on_message_recorded(message: object) -> None:
+        if getattr(message, "payload", None) != b"after-recovery":
+            return
+        confirmer = confirmers[0]
+        confirmation_results.extend(
+            (
+                confirmer(999),  # stale/future attempt
+                confirmer(1),  # exact post-publication edge
+                confirmer(1),  # duplicate edge
+            )
+        )
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "confirmed-recovery",
+        duration_seconds=30,
+        on_message_recorded=on_message_recorded,
+        recover_connection=lambda _attempt: "resume",
+        on_recovery_confirmer_ready=confirmers.append,
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    assert confirmation_results == [False, True, False]
+    assert summary["recovery_attempts"] == 1
+    assert summary["recovery_point_cloud_candidates"] == 1
+    assert summary["successful_recoveries"] == 1
+    assert len(summary["recovery_gaps"]) == 1
+    gap = summary["recovery_gaps"][0]
+    assert gap["gap_index"] == 1
+    assert gap["recovery_attempt"] == 1
+    assert gap["outcome"] == "recovered"
+    assert gap["ended_monotonic_ns"] >= gap["started_monotonic_ns"]
+    assert gap["duration_seconds"] >= 0
+
+    journal_path = tmp_path / "confirmed-recovery" / "mqtt.recovery.jsonl"
+    journal = [json.loads(line) for line in journal_path.read_text().splitlines()]
+    assert [record["record_type"] for record in journal] == [
+        "recovery_gap_started",
+        "recovery_gap_ended",
+    ]
+    assert journal[1]["gap_index"] == gap["gap_index"]
+    assert journal[1]["recovery_attempt"] == gap["recovery_attempt"]
+    assert journal[1]["outcome"] == gap["outcome"]
+    assert summary["artifact_hashes"]["recovery_gaps_jsonl_sha256"] == hashlib.sha256(
+        journal_path.read_bytes()
+    ).hexdigest()
+
+
+@pytest.mark.parametrize("wake_step", [3, 4])
+def test_owner_wake_enters_same_single_recovery_loop_before_or_with_paho_loss(
+    tmp_path: Path,
+    wake_step: int,
+) -> None:
+    first: FakeClient = (
+        FakeClient(payload=b"before-owner-wake")
+        if wake_step == 3
+        else ConnectionLostAfterMessageClient(payload=b"before-owner-wake")
+    )
+    second = FakeClient(topic="RealtimePointcloud", payload=b"after-owner-wake")
+    clients = iter((first, second))
+    wake_consumed = False
+    losses: list[str] = []
+    attempts: list[int] = []
+
+    def consume_owner_wake() -> str | None:
+        nonlocal wake_consumed
+        if not wake_consumed and first._step >= wake_step:  # noqa: SLF001
+            wake_consumed = True
+            return "camera-source-ended"
+        return None
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / f"owner-wake-{wake_step}",
+        duration_seconds=30,
+        on_connection_lost=losses.append,
+        consume_connection_recovery_request=consume_owner_wake,
+        recover_connection=lambda attempt: attempts.append(attempt) or "resume",
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    frames = list(
+        iter_capture_frames(tmp_path / f"owner-wake-{wake_step}" / "mqtt.raw.k1mqtt")
+    )
+    assert [frame.payload for frame in frames] == [
+        b"before-owner-wake",
+        b"after-owner-wake",
+    ]
+    assert wake_consumed is True
+    assert len(losses) == 1
+    assert attempts == [1]
+    assert first.disconnect_count == 1
+    assert second.disconnect_count == 1
+    assert summary["recovery_attempts"] == 1
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+    assert summary["message_count"] == 2
+
+
+def test_guarded_recovery_retries_transient_subscription_handshake_failure(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    transient = SubscriptionUnavailableClient()
+    recovered_client = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery")
+    clients = iter((first, transient, recovered_client))
+    attempts: list[int] = []
+    candidates: list[tuple[int, int]] = []
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "recovering-subscription",
+        duration_seconds=30,
+        recover_connection=lambda attempt: attempts.append(attempt) or "resume",
+        on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append(
+            (attempt, sequence)
+        ),
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    frames = list(
+        iter_capture_frames(tmp_path / "recovering-subscription" / "mqtt.raw.k1mqtt")
+    )
+    assert [frame.payload for frame in frames] == [b"before-loss", b"after-recovery"]
+    assert attempts == [1, 2]
+    assert candidates == [(2, 2)]
+    assert summary["recovery_attempts"] == 2
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+
+
+def test_guarded_recovery_suback_and_pose_without_point_cloud_stay_reconnecting(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    subscribed_only = SubscribedWithoutPointCloudClient()
+    recovered_client = SpatialSequenceClient(
+        [("lixel/application/report/lio_pcl", b"fresh-pcl", False)]
+    )
+    clients = iter((first, subscribed_only, recovered_client))
+    attempts: list[int] = []
+    candidates: list[tuple[int, int]] = []
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "suback-without-spatial-data",
+        duration_seconds=30,
+        recover_connection=lambda attempt: attempts.append(attempt) or "resume",
+        on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append(
+            (attempt, sequence)
+        ),
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    assert attempts == [1, 2]
+    assert candidates == [(2, 5)]
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+    assert summary["message_count"] == 5
+
+
+def test_guarded_recovery_arms_candidate_before_enqueueing_first_fresh_point_cloud(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    recovered_client = SpatialSequenceClient(
+        [
+            ("lixel/application/report/heartbeat", b"heartbeat", False),
+            ("lixel/application/report/lio_pcl", b"retained-pcl", True),
+            ("DeviceStatus", b"status", False),
+            ("RealtimePath", b"fresh-pose", False),
+            ("RealtimePointcloud", b"fresh-pcl", False),
+            ("RealtimePath", b"later-pose", False),
+        ]
+    )
+    clients = iter((first, recovered_client))
+    candidates: list[tuple[int, int]] = []
+    events: list[str] = []
+
+    def record_candidate(attempt: int, sequence: int) -> None:
+        candidates.append((attempt, sequence))
+        events.append(f"candidate:{attempt}:{sequence}")
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "fresh-point-cloud-recovery",
+        duration_seconds=30,
+        on_message_recorded=lambda message: events.append(
+            f"message:{message.payload.decode()}"
+        ),
+        recover_connection=lambda _attempt: "resume",
+        on_recovery_point_cloud_candidate=record_candidate,
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    assert candidates == [(1, 6)]
+    assert events.index("message:fresh-pose") < events.index("message:fresh-pcl")
+    assert events.index("candidate:1:6") < events.index("message:fresh-pcl")
+    assert events.count("candidate:1:6") == 1
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+    assert summary["message_count"] == 7
+
+
+def test_guarded_recovery_ignores_late_spatial_callback_from_retired_client(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    recovered_client = LateOldClientCallbackRecoveryClient(first)
+    clients = iter((first, recovered_client))
+    recorded_payloads: list[bytes] = []
+    candidates: list[tuple[int, int]] = []
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "late-old-client-callback",
+        duration_seconds=30,
+        on_message_recorded=lambda message: recorded_payloads.append(message.payload),
+        recover_connection=lambda _attempt: "resume",
+        on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append(
+            (attempt, sequence)
+        ),
+        _client_factory=lambda: cast(mqtt.Client, next(clients)),
+    )
+
+    assert candidates == [(1, 2)]
+    assert recorded_payloads == [b"before-loss", b"fresh-new-client-pcl"]
+    assert summary["message_count"] == 2
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_point_cloud_candidates"] == 1
+
+
+def test_failed_resubscribe_consumes_resume_before_later_standby(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient(payload=b"before-loss")
+    transient = SubscriptionUnavailableClient()
+    clients = iter((first, transient))
+    factory_calls = 0
+
+    def factory() -> mqtt.Client:
+        nonlocal factory_calls
+        factory_calls += 1
+        return cast(mqtt.Client, next(clients))
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "recovery-standby-after-resubscribe-failure",
+        duration_seconds=30,
+        recover_connection=lambda attempt: "resume" if attempt == 1 else "standby",
+        _client_factory=factory,
+    )
+
+    assert factory_calls == 2
+    assert summary["stop_reason"] == "recovery_standby"
+    assert summary["recovery_attempts"] == 2
+    assert summary["successful_recoveries"] == 0
+
+
+def test_guarded_recovery_can_stay_blocked_until_local_owner_finishes(
+    tmp_path: Path,
+) -> None:
+    first = ConnectionLostAfterMessageClient()
+    factory_calls = 0
+    stop_checks = 0
+
+    def factory() -> mqtt.Client:
+        nonlocal factory_calls
+        factory_calls += 1
+        return cast(mqtt.Client, first)
+
+    def should_stop() -> bool:
+        nonlocal stop_checks
+        stop_checks += 1
+        return stop_checks >= 6
+
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "blocked-capture",
+        duration_seconds=30,
+        should_stop=should_stop,
+        recover_connection=lambda _attempt: "blocked",
+        _client_factory=factory,
+    )
+
+    assert factory_calls == 1
+    assert summary["stop_reason"] == "external_stop"
+    assert summary["recovery_attempts"] == 1
+    assert summary["successful_recoveries"] == 0
+    assert summary["recovery_blocked"] is True
+
+
+def test_guarded_recovery_standby_is_truthful_non_error_completion(
+    tmp_path: Path,
+) -> None:
+    summary = capture_mqtt(
+        "192.168.1.50",
+        tmp_path / "standby-capture",
+        duration_seconds=30,
+        recover_connection=lambda _attempt: "standby",
+        _client_factory=lambda: cast(mqtt.Client, ConnectionLostAfterMessageClient()),
+    )
+
+    assert summary["stop_reason"] == "recovery_standby"
+    assert summary["message_count"] == 1
+    assert summary["recovery_attempts"] == 1
+    assert summary["successful_recoveries"] == 0
+
+
 def test_owner_seals_session_clock_after_all_producers_stop(tmp_path: Path) -> None:
    capture_dir = tmp_path / "capture"
    summary = capture_mqtt(