SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): stabilize repeated acquisition and live viewer recovery

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 01:55:12 +03:00
parent 37b8930527
commit 2a97cf28c0
28 changed files with 2776 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py
+8 -7
View File
@@ -163,13 +163,18 @@ async def read_wifi_status_once(
device_macos_uuid: str,
*,
timeout_seconds: float = 20.0,
rediscover: bool = False,
) -> WifiStatusReadResult:
"""Read the K1's current DHCP status over BLE without writing a characteristic."""
if timeout_seconds <= 0:
raise ValueError("timeout_seconds must be positive")
async with asyncio.timeout(timeout_seconds + 5.0):
device = discovered_device(device_macos_uuid)
# A CoreBluetooth handle retained by an earlier scan is an optimization,
# not durable connection state. Recovery after sleep, Wi-Fi transition,
# or a completed provisioning GATT session must rediscover the device
# instead of repeatedly opening a stale handle.
device = None if rediscover else discovered_device(device_macos_uuid)
if device is None:
device = await BleakScanner.find_device_by_address(
device_macos_uuid,
@@ -183,9 +188,7 @@ async def read_wifi_status_once(
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
service = client.services.get_service(SERVICE_UUID)
status_characteristic = client.services.get_characteristic(
STATUS_CHARACTERISTIC_UUID
)
status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID)
if service is None:
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
if status_characteristic is None:
@@ -193,9 +196,7 @@ async def read_wifi_status_once(
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
)
if status_characteristic.service_uuid != service.uuid:
raise ValueError(
"K1 status characteristic is attached to an unexpected service"
)
raise ValueError("K1 status characteristic is attached to an unexpected service")
if "read" not in status_characteristic.properties:
raise ValueError("Reviewed K1 status characteristic is not readable")
value = bytes(await client.read_gatt_char(status_characteristic))
src/k1link/device_plugins/xgrids_k1/facade.py
+704 -148
View File
File diff suppressed because it is too large Load Diff
src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py
+47 -5
View File
@@ -243,6 +243,9 @@ class ApplicationMqttTransportSnapshot:
latest_device_init_ready: bool | None
latest_system_error_code: int | None
latest_system_error_state: str | None
last_loop_result_code: int | None
last_loop_result_name: str | None
last_loop_phase: str | None
operation_keys_consumed: int
clean_session: bool = False
keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS
@@ -268,6 +271,9 @@ class ApplicationMqttTransportSnapshot:
"latest_device_init_ready": self.latest_device_init_ready,
"latest_system_error_code": self.latest_system_error_code,
"latest_system_error_state": self.latest_system_error_state,
"last_loop_result_code": self.last_loop_result_code,
"last_loop_result_name": self.last_loop_result_name,
"last_loop_phase": self.last_loop_phase,
"operation_keys_consumed": self.operation_keys_consumed,
"clean_session": self.clean_session,
"keepalive_seconds": self.keepalive_seconds,
@@ -348,6 +354,9 @@ class ReviewedApplicationMqttTransport:
self._latest_device_fault = False
self._latest_system_error_code: int | None = None
self._latest_system_error_state: str | None = None
self._last_loop_result_code: int | None = None
self._last_loop_result_name: str | None = None
self._last_loop_phase: str | None = None
def open(self) -> ApplicationMqttTransportSnapshot:
with self._lock:
@@ -562,7 +571,9 @@ class ReviewedApplicationMqttTransport:
except (OSError, RuntimeError, ValueError) as exc:
self._fail_after_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_after_publish(
self._record_loop_failure(result, phase="maintain-open")
)
self._discard_allowed_responses(allowed)
def close(self) -> None:
@@ -649,6 +660,9 @@ class ReviewedApplicationMqttTransport:
latest_device_init_ready=self._latest_device_init_ready,
latest_system_error_code=self._latest_system_error_code,
latest_system_error_state=self._latest_system_error_state,
last_loop_result_code=self._last_loop_result_code,
last_loop_result_name=self._last_loop_result_name,
last_loop_phase=self._last_loop_phase,
operation_keys_consumed=len(self._consumed_operation_keys),
)
@@ -879,9 +893,13 @@ class ReviewedApplicationMqttTransport:
self._fail_after_publish("control MQTT network loop failed", exc)
self._fail_before_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
failure = self._record_loop_failure(
result,
phase="post-publish" if post_publish else "connect-subscribe",
)
if post_publish:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_before_publish("control MQTT network loop returned an error")
self._fail_after_publish(failure)
self._fail_before_publish(failure)
def _service_once(self, *, post_publish: bool) -> None:
with self._lock:
@@ -898,9 +916,33 @@ class ReviewedApplicationMqttTransport:
self._fail_after_publish("control MQTT network loop failed", exc)
self._fail_before_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
failure = self._record_loop_failure(
result,
phase="post-publish-drain" if post_publish else "pre-publish-drain",
)
if post_publish:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_before_publish("control MQTT network loop returned an error")
self._fail_after_publish(failure)
self._fail_before_publish(failure)
def _record_loop_failure(
self,
result: int,
*,
phase: str,
) -> str:
result_code = int(result)
try:
result_name = mqtt.error_string(result_code)
except (TypeError, ValueError):
result_name = "unknown MQTT error"
with self._lock:
self._last_loop_result_code = result_code
self._last_loop_result_name = result_name
self._last_loop_phase = phase
return (
"control MQTT network loop returned an error "
f"(phase={phase}, result={result_code}: {result_name})"
)
def _drain_responses(
self,
src/k1link/device_plugins/xgrids_k1/protocol/application_session.py
+196 -5
View File
@@ -62,6 +62,7 @@ logger = logging.getLogger(__name__)
TransportFactory = Callable[[str], ReviewedApplicationMqttTransport]
ScanningObserver = Callable[[], None]
@dataclass(frozen=True, slots=True)
@@ -97,10 +98,13 @@ class InteractiveApplicationControlSession:
*,
transport_factory: TransportFactory = ReviewedApplicationMqttTransport,
epoch_seconds: Callable[[], int] = lambda: int(time.time()),
scanning_observer: ScanningObserver | None = None,
) -> None:
self._authority_loader = authority_loader
self._transport_factory = transport_factory
self._epoch_seconds = epoch_seconds
self._scanning_observer = scanning_observer
self._scanning_observer_errors = 0
self._lock = threading.RLock()
self._phase: ApplicationControlPhase = "idle"
self._host: str | None = None
@@ -215,6 +219,55 @@ class InteractiveApplicationControlSession:
transport.close()
return self.snapshot()
def retire_for_network_change(self) -> dict[str, object]:
"""Retire local control ownership for one new explicit network action.
A correlated STOP followed by a network-loop loss in SCAN_STOPPING is
not enough evidence to authorize another START. It is enough to retire
the dead local socket when the operator explicitly chooses a new
network route. Pre-START failures already classified safe for a fresh
click are admissible here for the same reason: no modeling command was
attempted.
"""
with self._lock:
failure = self._failure
failed_network_change_admissible = (
self._phase == "failed"
and failure is not None
and (
failure.get("safe_to_retry") is True
or failure.get("network_change_admissible") is True
)
)
if self._phase not in {"idle", "completed", "closed"} and not (
failed_network_change_admissible
):
raise ApplicationAcceptanceError(
"control session cannot be retired for a network change"
)
if not self._worker_retired_locked():
raise ApplicationAcceptanceError(
"control session worker is still retiring"
)
previous_phase = self._phase
previous_reason = (
self._json_string(failure.get("reason_code"))
if failure is not None
else None
)
self._reset_locked()
if previous_phase != "idle":
logger.info(
"K1 local control session retired for an explicit network change",
extra={
"event_code": "k1_control_session_retired_for_network_change",
"reason_code": previous_reason,
"automatic_retry": False,
},
)
return self.snapshot()
def close(self) -> None:
"""Stop the process-owned socket without ever inventing a device STOP."""
@@ -247,6 +300,7 @@ class InteractiveApplicationControlSession:
"scripted_transitions": False,
"automatic_retry": False,
"outcome_unknown": self._outcome_unknown,
"scanning_observer_errors": self._scanning_observer_errors,
"failure": dict(self._failure) if self._failure is not None else None,
"dialogue": (
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
@@ -310,6 +364,7 @@ class InteractiveApplicationControlSession:
checkpoint=start_checkpoint,
)
self._set_phase("scanning")
self._notify_scanning_observer()
executor.maintain_active_until_stop_requested(self._stop_requested.is_set)
stop_confirmation = self._stop_request()
@@ -412,10 +467,52 @@ class InteractiveApplicationControlSession:
)
== 1
)
safe_to_retry = not outcome_unknown and (
not transport_created
or (transport_snapshot_available and publish_attempts == 0)
or correlated_read_only_profile_mismatch
status_reconciled_prestart_failure = (
outcome_unknown
and modeling_command_attempted is False
and transport_snapshot_available
and not diagnostic_evidence_unavailable
and self._json_int_or_none(
transport_snapshot.get("device_status_reports")
)
not in {None, 0}
and self._json_string(
transport_snapshot.get("latest_device_session_state")
)
== "ready"
and transport_snapshot.get("latest_device_project_bound") is False
and self._json_int_or_none(
transport_snapshot.get("latest_system_error_code")
)
is None
)
stop_acknowledged_network_change = (
outcome_unknown
and failure_reason_code == "mqtt_network_loop_failed"
and failed_phase == "awaiting-standby-confirmation"
and dialogue_snapshot_available
and transport_snapshot_available
and dialogue_snapshot.get("stop_attempted") is True
and dialogue_snapshot.get("stop_complete") is True
and dialogue_stage == "stop-acknowledged"
and modeling_command_attempted is True
and self._json_string(
transport_snapshot.get("latest_device_session_state")
)
== "scan_stopping"
and transport_snapshot.get("latest_device_project_bound") is True
and self._json_int_or_none(
transport_snapshot.get("latest_system_error_code")
)
is None
)
safe_to_retry = status_reconciled_prestart_failure or (
not outcome_unknown
and (
not transport_created
or (transport_snapshot_available and publish_attempts == 0)
or correlated_read_only_profile_mismatch
)
)
self._failure = {
"code": type(exc).__name__,
@@ -451,6 +548,37 @@ class InteractiveApplicationControlSession:
else None
),
"safe_to_retry": safe_to_retry,
**(
{
"status_reconciliation": {
"device_session_state": "ready",
"device_project_bound": False,
"system_error_code": None,
"decision": "safe-explicit-prestart-retry",
"automatic_retry": False,
}
}
if status_reconciled_prestart_failure
else {}
),
**(
{
"network_change_admissible": True,
"network_change_reconciliation": {
"device_session_state": "scan_stopping",
"device_project_bound": True,
"system_error_code": None,
"stop_complete": True,
"standby_confirmed": False,
"decision": (
"explicit-network-change-only-after-acknowledged-stop"
),
"automatic_retry": False,
},
}
if stop_acknowledged_network_change
else {}
),
}
self._outcome_unknown = outcome_unknown
self._dialogue_snapshot = dialogue_snapshot or None
@@ -462,7 +590,8 @@ class InteractiveApplicationControlSession:
"publish_attempts=%s modeling_command_attempted=%s "
"diagnostic_snapshot_unavailable=%s "
"diagnostic_evidence_unavailable=%s outcome_unknown=%s "
"safe_to_retry=%s",
"safe_to_retry=%s status_reconciliation=%s "
"network_change_reconciliation=%s",
type(exc).__name__,
failure_reason_code,
failed_phase,
@@ -474,6 +603,51 @@ class InteractiveApplicationControlSession:
diagnostic_evidence_unavailable,
outcome_unknown,
safe_to_retry,
(
"safe-explicit-prestart-retry"
if status_reconciled_prestart_failure
else None
),
(
"explicit-network-change-only-after-acknowledged-stop"
if stop_acknowledged_network_change
else None
),
extra={
"event_code": "k1_application_control_session_failed",
"reason_code": failure_reason_code,
"failed_phase": failed_phase,
"dialogue_stage": dialogue_stage,
"transport_state": self._json_string(
transport_snapshot.get("state")
),
"publish_attempts": publish_attempts,
"modeling_command_attempted": modeling_command_attempted,
"outcome_unknown": outcome_unknown,
"safe_to_retry": safe_to_retry,
"status_reconciliation": (
"safe-explicit-prestart-retry"
if status_reconciled_prestart_failure
else None
),
"network_change_admissible": (
stop_acknowledged_network_change
),
"network_change_reconciliation": (
"explicit-network-change-only-after-acknowledged-stop"
if stop_acknowledged_network_change
else None
),
"mqtt_loop_result_code": self._json_int_or_none(
transport_snapshot.get("last_loop_result_code")
),
"mqtt_loop_result_name": self._json_string(
transport_snapshot.get("last_loop_result_name")
),
"mqtt_loop_phase": self._json_string(
transport_snapshot.get("last_loop_phase")
),
},
)
finally:
final_dialogue_snapshot: dict[str, object] | None = None
@@ -552,6 +726,23 @@ class InteractiveApplicationControlSession:
with self._lock:
self._set_phase_locked(phase)
def _notify_scanning_observer(self) -> None:
observer = self._scanning_observer
if observer is None:
return
try:
observer()
except Exception:
with self._lock:
self._scanning_observer_errors += 1
logger.exception(
"K1 confirmed SCANNING, but its acquisition observer failed",
extra={
"event_code": "k1_application_scanning_observer_failed",
"automatic_retry": False,
},
)
@staticmethod
def _executor_snapshot_safely(
executor: PhysicalAcceptanceDialogueExecutor | None,
src/k1link/device_plugins/xgrids_k1/viewer/runtime.py
+26 -25
View File
@@ -226,7 +226,7 @@ class VisualizationRuntime:
raise RuntimeError("поток не завершился за отведённое время; повторите остановку")
def close(self, *, wait_seconds: float = 5.0) -> None:
"""Stop the active source and release the process-wide visual bridge."""
"""Stop the active source and release its acquisition-scoped bridge."""
with self._lock:
self._closed = True
thread = self._thread
@@ -429,24 +429,25 @@ class VisualizationRuntime:
bridge: RerunBridge | None = None
try:
with self._lock:
bridge = self._bridge
if self._closed:
publisher_aborted.set()
if publisher_aborted.is_set():
publisher_ready.set()
return
if bridge is None:
candidate = self._bridge_factory(
grpc_port=self._grpc_port,
metrics=self._metrics,
settings_provider=self._current_scene_settings,
)
bridge = candidate
with self._lock:
if self._closed:
publisher_aborted.set()
else:
self._bridge = candidate
candidate = self._bridge_factory(
grpc_port=self._grpc_port,
metrics=self._metrics,
settings_provider=self._current_scene_settings,
)
bridge = candidate
with self._lock:
if self._closed:
publisher_aborted.set()
else:
# Every acquisition owns a fresh Rerun recording/store.
# Reusing a RecordingStream across independent scans can
# leave a later WebViewer without a StoreInfo event.
self._bridge = candidate
if publisher_aborted.is_set():
publisher_ready.set()
return
@@ -520,19 +521,17 @@ class VisualizationRuntime:
finally:
if bridge is not None:
with self._lock:
close_bridge = self._closed and (
self._bridge is bridge or publisher_aborted.is_set()
)
if close_bridge and self._bridge is bridge:
if self._bridge is bridge:
self._bridge = None
self._rerun_grpc_url = None
if close_bridge:
try:
bridge.close()
except BaseException as exc:
# Process shutdown must not become a false successful
# idle state when the native bridge failed to close.
publisher_error.append(exc)
try:
bridge.close()
except BaseException as exc:
# A failed native disconnect must not become a false
# successful idle state or leak port 9876 into the next
# independent acquisition.
publisher_error.append(exc)
self._notify()
def join_publisher() -> None:
# Never orphan a publisher: the session thread remains its owner.
@@ -596,6 +595,7 @@ class VisualizationRuntime:
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
self._rerun_grpc_url = None
self._notify()
def _finish_error(self, message: str) -> None:
@@ -605,6 +605,7 @@ class VisualizationRuntime:
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
self._rerun_grpc_url = None
self._notify()
def _current_scene_settings(self) -> RerunSceneSettings: